Initial commit
This commit is contained in:
commit
49cb994257
|
@ -0,0 +1,3 @@
|
||||||
|
raw
|
||||||
|
*.cnf
|
||||||
|
zscaler.txt
|
|
@ -0,0 +1,51 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="API Call to collect data")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
def print_dict(dict):
|
||||||
|
for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
|
||||||
|
print ("{:<35s}{:5d}".format(key, value))
|
||||||
|
|
||||||
|
base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
|
||||||
|
|
||||||
|
req = urllib.request.Request(base_url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_content = json.loads(content)
|
||||||
|
|
||||||
|
domains = collections.Counter()
|
||||||
|
categories = collections.Counter()
|
||||||
|
|
||||||
|
for i in range (0, len (json_content['data'])):
|
||||||
|
domain = json_content["data"][i]["domain"]
|
||||||
|
ccl = json_content["data"][i]["ccl"]
|
||||||
|
category = json_content["data"][i]["category"]
|
||||||
|
domains[domain] += 1
|
||||||
|
categories[category][count] += 1
|
||||||
|
|
||||||
|
|
||||||
|
#print ("===== Domains =====")
|
||||||
|
#print_dict(domains)
|
||||||
|
|
||||||
|
|
||||||
|
print ("\n===== Categories =====")
|
||||||
|
print_dict(categories)
|
||||||
|
|
|
@ -0,0 +1,40 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="API Call to collect data")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
def print_dict(dict, json_content):
|
||||||
|
for key, value in sorted(dict.items(), key = itemgetter(1), reverse = True):
|
||||||
|
print ("{:<35s}{:5d}".format(key, value))
|
||||||
|
|
||||||
|
base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
|
||||||
|
req = urllib.request.Request(base_url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_content = json.loads(content)
|
||||||
|
|
||||||
|
domains = Counter(data['domain'] for data in json_content['data'])
|
||||||
|
categories = Counter(data['category'] for data in json_content['data'])
|
||||||
|
|
||||||
|
print ("==== Domains ===")
|
||||||
|
print_dict (domains, json_content)
|
||||||
|
print ("\n")
|
||||||
|
print ("==== Categories ===")
|
||||||
|
print_dict (categories)
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191028
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
|
||||||
|
parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
|
||||||
|
parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
rows = args.rows
|
||||||
|
show = args.show
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
base_url = "https://{}.goskope.com/api/v1/events?token={}&type=application&timeperiod={}".format(tenant, token, timeperiod)
|
||||||
|
|
||||||
|
req = urllib.request.Request(base_url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_content = json.loads(content)
|
||||||
|
|
||||||
|
domain_count = Counter()
|
||||||
|
domain_category = {}
|
||||||
|
category_count = Counter()
|
||||||
|
rows = None if rows == 0 else rows
|
||||||
|
|
||||||
|
for i in range (0, len (json_content['data'])):
|
||||||
|
domain = json_content["data"][i]["domain"]
|
||||||
|
ccl = json_content["data"][i]["ccl"]
|
||||||
|
category = json_content["data"][i]["category"]
|
||||||
|
domain_count[domain] += 1
|
||||||
|
domain_category[domain] = category
|
||||||
|
category_count[category] += 1
|
||||||
|
|
||||||
|
top_domains = domain_count.most_common(rows)
|
||||||
|
print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_domains:
|
||||||
|
print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
|
||||||
|
|
||||||
|
print ("")
|
||||||
|
if show:
|
||||||
|
top_categories = category_count.most_common()
|
||||||
|
print ("{:<40s}{:>5s}".format("Category", "Hits"))
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_categories:
|
||||||
|
print ("{:<40s}{:5d}".format(i[0], i[1]))
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191028
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 604800)")
|
||||||
|
parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
|
||||||
|
parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
rows = args.rows
|
||||||
|
show = args.show
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
|
||||||
|
|
||||||
|
req = urllib.request.Request(base_url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_content = json.loads(content)
|
||||||
|
|
||||||
|
#site = {'data': []}
|
||||||
|
site = collections.defaultdict(list);
|
||||||
|
|
||||||
|
rows = None if rows == 0 else rows
|
||||||
|
|
||||||
|
for i in range (0, len (json_content['data'])):
|
||||||
|
json_site = json_content["data"][i]["site"]
|
||||||
|
json_domain = json_content["data"][i]["domain"]
|
||||||
|
|
||||||
|
if json_domain not in site[json_site]:
|
||||||
|
site[json_site].append(json_domain)
|
||||||
|
#print (json_site, "-", json_domain)
|
||||||
|
|
||||||
|
print (site)
|
||||||
|
|
||||||
|
for key, value in sorted(site.items(), key = itemgetter(0), reverse = False):
|
||||||
|
print ("{:<35s}".format(key), end="")
|
||||||
|
for i in value:
|
||||||
|
print ("{},".format(i), end="")
|
||||||
|
print ("")
|
||||||
|
|
||||||
|
|
||||||
|
#top_domains = domain_count.most_common(rows)
|
||||||
|
#print ("{:<40s}{:>5s} - {}".format("Domain", "Hits", "Category"))
|
||||||
|
#print ("################################################################################")
|
||||||
|
#for i in top_domains:
|
||||||
|
#print ("{:<40s}{:5d} - {}".format(i[0], i[1], domain_category[i[0]]))
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191028
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
|
||||||
|
parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
|
||||||
|
parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
|
||||||
|
parser.add_argument("-d", "--debug", action='store_true', help="debug")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
rows = args.rows
|
||||||
|
show = args.show
|
||||||
|
debug = args.debug
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
domain_count = Counter()
|
||||||
|
domain_category = {}
|
||||||
|
category_count = Counter()
|
||||||
|
rows = None if rows == 0 else rows
|
||||||
|
|
||||||
|
def get_json(type):
|
||||||
|
domain = "goskope.com"
|
||||||
|
url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
|
||||||
|
req = urllib.request.Request(url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_data = json.loads(content)
|
||||||
|
if debug: print (json_data)
|
||||||
|
return(json_data)
|
||||||
|
|
||||||
|
json_content = get_json("application")
|
||||||
|
for i in range (0, len (json_content['data'])):
|
||||||
|
domain = json_content["data"][i]["domain"]
|
||||||
|
ccl = json_content["data"][i]["ccl"]
|
||||||
|
category = json_content["data"][i]["category"]
|
||||||
|
domain_count[domain] += 1
|
||||||
|
domain_category[domain] = category
|
||||||
|
category_count[category] += 1
|
||||||
|
|
||||||
|
top_domains = domain_count.most_common(rows)
|
||||||
|
print (f"{'Domain':<40s}{'Hits':>5s} - Category")
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_domains:
|
||||||
|
print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]}")
|
||||||
|
|
||||||
|
print ("")
|
||||||
|
if show:
|
||||||
|
top_categories = category_count.most_common()
|
||||||
|
print (f"{'Category':<40s}{'Hits':>5s}")
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_categories:
|
||||||
|
print (f"{i[0]:<40s}{i[1]:5d}")
|
|
@ -0,0 +1,89 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191107
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
import re
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenant API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
|
||||||
|
parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
|
||||||
|
parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
|
||||||
|
parser.add_argument("-d", "--debug", action='store_true', help="debug")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
records = args.records
|
||||||
|
verbose = args.verbose
|
||||||
|
debug = args.debug
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
cursor_up = '\x1b[1A'
|
||||||
|
erase_line = '\x1b[2K'
|
||||||
|
cct_list = ["Cloud Storage", "Webmail"]
|
||||||
|
ccl_list = ["low", "poor"]
|
||||||
|
whitelist = re.compile("bla")
|
||||||
|
ioc_list = []
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print("Using Categories: ", end='', flush=True)
|
||||||
|
print(", ".join(map(str,cct_list)))
|
||||||
|
print("Using Rating: ", end='', flush=True)
|
||||||
|
print(", ".join(map(str,ccl_list)))
|
||||||
|
print(f"Applying Whitelist for: {whitelist.pattern}")
|
||||||
|
|
||||||
|
def get_json(type):
|
||||||
|
domain = "goskope.com"
|
||||||
|
url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
|
||||||
|
req = urllib.request.Request(url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_data = json.loads(content)
|
||||||
|
if debug: print (json_data)
|
||||||
|
return(json_data)
|
||||||
|
|
||||||
|
print()
|
||||||
|
print("Processing...", end='', flush=True)
|
||||||
|
json_content = get_json("page")
|
||||||
|
sys.stdout.write(cursor_up)
|
||||||
|
sys.stdout.write(erase_line)
|
||||||
|
print()
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print(f"{'#':>4} {'Domain':<50s} Confidence")
|
||||||
|
print("#######################################################################")
|
||||||
|
|
||||||
|
for index, data in enumerate(json_content['data']):
|
||||||
|
if not "domain" in data:
|
||||||
|
domain = urlparse(data["url"]).netloc
|
||||||
|
else:
|
||||||
|
domain = data["domain"]
|
||||||
|
if whitelist.search(domain):
|
||||||
|
continue
|
||||||
|
if data["category"] in cct_list:
|
||||||
|
if data["ccl"] in ccl_list:
|
||||||
|
if domain not in ioc_list:
|
||||||
|
i += 1
|
||||||
|
if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
|
||||||
|
ioc_list.append(domain)
|
||||||
|
if i == records:
|
||||||
|
break
|
||||||
|
|
||||||
|
if verbose: print()
|
||||||
|
print(", ".join(map(str,ioc_list)))
|
|
@ -0,0 +1,95 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191107
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Collect all page events from Netskope API and process domains by category and confidence")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenant API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
|
||||||
|
parser.add_argument("-r", "--records", type=int, default=100, help="# of records (default: 100)")
|
||||||
|
parser.add_argument("-v", "--verbose", action='store_true', help="verbose")
|
||||||
|
parser.add_argument("-d", "--debug", action='store_true', help="print raw json data")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
records = args.records
|
||||||
|
verbose = args.verbose
|
||||||
|
debug = args.debug
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
cct_list = ["Cloud Storage", "Webmail"]
|
||||||
|
ccl_list = ["low", "poor"]
|
||||||
|
whitelist = re.compile("yahoo")
|
||||||
|
ioc_list = []
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print("Using Categories: ", end='', flush=True)
|
||||||
|
print(", ".join(map(str,cct_list)))
|
||||||
|
print("Using Rating: ", end='', flush=True)
|
||||||
|
print(", ".join(map(str,ccl_list)))
|
||||||
|
print(f"Applying Whitelist: {whitelist.pattern}")
|
||||||
|
print()
|
||||||
|
print(f"{'#':>4} {'Domain':<50s} Confidence")
|
||||||
|
print("#######################################################################")
|
||||||
|
|
||||||
|
def get_json(type):
|
||||||
|
domain = "goskope.com"
|
||||||
|
url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
|
||||||
|
req = urllib.request.Request(url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_data = json.loads(content)
|
||||||
|
if debug: print (json_data)
|
||||||
|
return(json_data)
|
||||||
|
|
||||||
|
def parse_json(json_content):
|
||||||
|
i = 0
|
||||||
|
for index, data in enumerate(json_content['data']):
|
||||||
|
if not "domain" in data:
|
||||||
|
domain = urllib.parse.urlparse(data["url"]).netloc
|
||||||
|
else:
|
||||||
|
domain = data["domain"]
|
||||||
|
if whitelist.search(domain):
|
||||||
|
continue
|
||||||
|
if data["category"] in cct_list:
|
||||||
|
if data["ccl"] in ccl_list:
|
||||||
|
if domain not in ioc_list:
|
||||||
|
i += 1
|
||||||
|
if verbose: print(f"{i:>4}) {domain:<50s} {data['ccl']}")
|
||||||
|
ioc_list.append(domain)
|
||||||
|
|
||||||
|
return ioc_list
|
||||||
|
#domain_list = ", ".join(map(str,ioc_list[:records]))
|
||||||
|
#return domain_list
|
||||||
|
|
||||||
|
json = get_json("page")
|
||||||
|
print(parse_json(json))
|
|
@ -0,0 +1,80 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191107
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
import urllib.parse
|
||||||
|
import requests
|
||||||
|
|
||||||
|
NTSKP_TENANT = 'https://astrazeneca.eu.goskope.com'
|
||||||
|
NTSKP_TOKEN = '604d0a3b26ea9b22c3ec42130ebbfa8e'
|
||||||
|
NTSKP_PERIOD = '2592000'
|
||||||
|
cct_list = ["Cloud Storage", "Webmail"]
|
||||||
|
ccl_list = ["low", "poor"]
|
||||||
|
whitelist = re.compile("yahoo")
|
||||||
|
ioc_list = []
|
||||||
|
|
||||||
|
ZS_MAX_DOMAINS = 2
|
||||||
|
headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': 'Netskope_ZscalerImporter1.0'}
|
||||||
|
PROXY=''
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
|
logging = logging.getLogger('zsc')
|
||||||
|
|
||||||
|
def ntskp_get_domains(headers):
|
||||||
|
uri = f"{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}"
|
||||||
|
try:
|
||||||
|
r = requests.get(uri, headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error('Error: ' + str(e))
|
||||||
|
sys.exit(1)
|
||||||
|
json = r.json()
|
||||||
|
limit = (len(json['data']))
|
||||||
|
|
||||||
|
for item in json['data']:
|
||||||
|
if not "domain" in item:
|
||||||
|
domain = urllib.parse.urlparse(item['url']).netloc
|
||||||
|
else:
|
||||||
|
domain = item['domain']
|
||||||
|
if whitelist.search(domain):
|
||||||
|
continue
|
||||||
|
if item['category'] in cct_list:
|
||||||
|
if item['ccl'] in ccl_list:
|
||||||
|
if domain not in ioc_list:
|
||||||
|
print(f"{domain:<50s} {item['ccl']}")
|
||||||
|
endtime = item['timestamp']
|
||||||
|
ioc_list.append(domain)
|
||||||
|
print(limit)
|
||||||
|
print(endtime)
|
||||||
|
starttime = endtime - (10 * 60)
|
||||||
|
print(ioc_list[:ZS_MAX_DOMAINS])
|
||||||
|
return ioc_list[:ZS_MAX_DOMAINS]
|
||||||
|
|
||||||
|
|
||||||
|
ntskp_get_domains(headers)
|
||||||
|
|
||||||
|
now = int(time.time() * 1000)
|
||||||
|
print(now)
|
||||||
|
#print(str(time.ctime(int(time.time()))))
|
|
@ -0,0 +1,80 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191028
|
||||||
|
#
|
||||||
|
# Collects all the page events, counts all the domain hits and category hits
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Get all events from Netskope API", epilog="2019 (c) Netskope")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name (eg. ams.eu)")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='86400', help="Timeperiod 3600 | 86400 | 604800 | 2592000 (default: 86400)")
|
||||||
|
parser.add_argument("-r", "--rows", type=int, default='0', help="Number of rows (default display all)")
|
||||||
|
parser.add_argument("-s", "--show", action='store_true', help="Show category hits")
|
||||||
|
parser.add_argument("-d", "--debug", action='store_true', help="debug")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
rows = args.rows
|
||||||
|
show = args.show
|
||||||
|
debug = args.debug
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
domain_count = Counter()
|
||||||
|
domain_category = {}
|
||||||
|
domain_ccl = {}
|
||||||
|
domain_cci = {}
|
||||||
|
category_count = Counter()
|
||||||
|
rows = None if rows == 0 else rows
|
||||||
|
|
||||||
|
def get_json(type):
|
||||||
|
domain = "goskope.com"
|
||||||
|
url = f"https://{tenant}.{domain}/api/v1/events?token={token}&type={type}&timeperiod={timeperiod}"
|
||||||
|
req = urllib.request.Request(url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_data = json.loads(content)
|
||||||
|
if debug: print (json_data)
|
||||||
|
print(json.dumps(json_data, indent=4, sort_keys=True))
|
||||||
|
return(json_data)
|
||||||
|
|
||||||
|
json_content = get_json("page")
|
||||||
|
for i in range (0, len (json_content['data'])):
|
||||||
|
domain = json_content["data"][i]["domain"]
|
||||||
|
ccl = json_content["data"][i]["ccl"]
|
||||||
|
category = json_content["data"][i]["category"]
|
||||||
|
#ccl = json_content["data"][i]["ccl"]
|
||||||
|
cci = json_content["data"][i]["cci"]
|
||||||
|
domain_count[domain] += 1
|
||||||
|
domain_category[domain] = category
|
||||||
|
domain_ccl[domain] = ccl
|
||||||
|
domain_cci[domain] = cci
|
||||||
|
category_count[category] += 1
|
||||||
|
|
||||||
|
top_domains = domain_count.most_common(rows)
|
||||||
|
print (f"{'Domain':<40s}{'Hits':>5s} - Category")
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_domains:
|
||||||
|
print (f"{i[0]:<40s}{i[1]:5d} - {domain_category[i[0]]} - {domain_ccl[i[0]]}")
|
||||||
|
|
||||||
|
print ("")
|
||||||
|
if show:
|
||||||
|
top_categories = category_count.most_common()
|
||||||
|
print (f"{'Category':<40s}{'Hits':>5s}")
|
||||||
|
print ("################################################################################")
|
||||||
|
for i in top_categories:
|
||||||
|
print (f"{i[0]:<40s}{i[1]:5d}")
|
|
@ -0,0 +1,91 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import re
|
||||||
|
import logging
|
||||||
|
import urllib.parse
|
||||||
|
import requests
|
||||||
|
import configparser
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
|
||||||
|
if not os.path.isfile(CONFIG_FILE):
|
||||||
|
logging.error(f"The config file {CONFIG_FILE} doesn't exist")
|
||||||
|
sys.exit(1)
|
||||||
|
config = configparser.RawConfigParser()
|
||||||
|
config.read(CONFIG_FILE)
|
||||||
|
NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
|
||||||
|
NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
|
||||||
|
NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
|
||||||
|
NTSKP_SCORE = config.get('netskope', 'NTSKP_SCORE')
|
||||||
|
NTSKP_CATEGORIES = config.get('netskope', 'NTSKP_CATEGORIES')
|
||||||
|
NTSKP_CONFIDENCE = config.get('netskope', 'NTSKP_CONFIDENCE')
|
||||||
|
PROXY = config.get('general', 'PROXY')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
# Use a custom user-agent string
|
||||||
|
UA_STRING = 'NetskopeAPICollector1.0'
|
||||||
|
|
||||||
|
# Set logging.INFO to logging.DEBUG for debug information
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
logging = logging.getLogger('NetskopeAPICollector')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
def ntskp_get_domains(headers):
|
||||||
|
skip = 0
|
||||||
|
filename = f"/home/mischa/netskope/api-{datetime.now().strftime('%Y%m%d')}.txt"
|
||||||
|
logging.info(f"File {filename} created")
|
||||||
|
ssl_session = requests.Session()
|
||||||
|
logging.debug(f"{ssl_session}")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
uri = f'{NTSKP_TENANT}/api/v1/events?token={NTSKP_TOKEN}&type=page&timeperiod={NTSKP_PERIOD}&skip={skip}'
|
||||||
|
try:
|
||||||
|
r = ssl_session.get(uri, headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
json = r.json()
|
||||||
|
#if json['data']:
|
||||||
|
if 'data' in json:
|
||||||
|
if len(json['data']) <= 5000:
|
||||||
|
skip += 5000
|
||||||
|
filter_file = open(filename, "a")
|
||||||
|
logging.debug(f"File {filename} opened")
|
||||||
|
for item in json['data']:
|
||||||
|
if not 'domain' in item:
|
||||||
|
domain = urllib.parse.urlparse(item['url']).netloc
|
||||||
|
else:
|
||||||
|
domain = item['domain']
|
||||||
|
|
||||||
|
#if NTSKP_SAFELIST.search(domain):
|
||||||
|
#print(domain)
|
||||||
|
|
||||||
|
#if item['ccl'] in NTSKP_CONFIDENCE:
|
||||||
|
utctime = datetime.utcfromtimestamp(item['timestamp']).strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
filter_file.write(f"{utctime},{domain},{item['cci']},{item['category']},{item['ccl']},{item['user']}\n")
|
||||||
|
filter_file.close()
|
||||||
|
logging.debug(f"File {filename} closed")
|
||||||
|
logging.debug(f"Next request, skip: {skip}")
|
||||||
|
else:
|
||||||
|
logging.info(f"No more data to collect")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logging.info(f"No more data to collect")
|
||||||
|
break
|
||||||
|
if skip == 500000:
|
||||||
|
logging.info(f"Reached limit")
|
||||||
|
break
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
|
||||||
|
ntskp_get_domains(request_headers)
|
|
@ -0,0 +1,86 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use File::Temp;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $response = HTTP::Tiny->new->get($uri);
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
my %files;
|
||||||
|
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
|
||||||
|
$files{$tmp_file} = $widget->{'name'};
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = HTTP::Tiny->new->get($uri);
|
||||||
|
open my $fh_out, ">", $tmp_file;
|
||||||
|
print $fh_out $response->{'content'};
|
||||||
|
close $fh_out;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $out_email = "azblocklist.csv";
|
||||||
|
my $out_zscaler = "zscaler.txt";
|
||||||
|
open my $fh_email, ">", $out_email;
|
||||||
|
open my $fh_zscaler, ">", $out_zscaler;
|
||||||
|
|
||||||
|
for my $item (keys %files) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh, "<", $item;
|
||||||
|
my $header = $csv->getline($fh);
|
||||||
|
|
||||||
|
print "$files{$item}\n";
|
||||||
|
print $fh_email "$files{$item}\n";
|
||||||
|
|
||||||
|
while (my $row = $csv->getline($fh)) {
|
||||||
|
last if ($count == 30);
|
||||||
|
if ($row->[1] =~ m/,/) {
|
||||||
|
my @domains = split "," , $row->[1];
|
||||||
|
for my $domain (@domains) {
|
||||||
|
print "$domain,";
|
||||||
|
print $fh_email "$domain,";
|
||||||
|
print $fh_zscaler "$domain\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
print "$row->[1],";
|
||||||
|
print $fh_email "$row->[1],";
|
||||||
|
print $fh_zscaler "$row->[1]\n";
|
||||||
|
}
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
print "\n";
|
||||||
|
print $fh_email "\n";
|
||||||
|
close $fh;
|
||||||
|
unlink $item;
|
||||||
|
}
|
||||||
|
close $fh_email;
|
||||||
|
close $fh_zscaler;
|
||||||
|
|
||||||
|
my $msg = MIME::Lite->new(
|
||||||
|
From => 'mischa@high5.nl',
|
||||||
|
To => 'mischa@netskope.com',
|
||||||
|
Cc => 'mischa@high5.nl',
|
||||||
|
Subject => 'AztraZeneca Netskope Blocklist',
|
||||||
|
Type => 'TEXT',
|
||||||
|
Data => "Domains pushed to Zscaler for blocking\n\n"
|
||||||
|
);
|
||||||
|
$msg->attach(
|
||||||
|
Type => 'text/csv',
|
||||||
|
Path => $out_email,
|
||||||
|
Filename => $out_email
|
||||||
|
);
|
||||||
|
$msg->send('smtp','mail.high5.nl', Debug=>0);
|
||||||
|
unlink $out_email;
|
|
@ -0,0 +1,44 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# OPLPUploader.sh - Version 1.0 - 20200113
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# find ${LOCALDIR} -type f -name "*.csv" -maxdepth 1 | sort -V
|
||||||
|
|
||||||
|
HOST="ftp://"
|
||||||
|
LOCALDIR="/tmp/files"
|
||||||
|
REMOTEDIR="/nslogs/user/upload/custom-ASML_SplunkCurlv1/"
|
||||||
|
USER=''
|
||||||
|
PASS=''
|
||||||
|
GLOB='*.csv'
|
||||||
|
LOG="/tmp/script.log"
|
||||||
|
|
||||||
|
if [ -d ${LOCALDIR} ]; then
|
||||||
|
cd ${LOCALDIR}
|
||||||
|
else
|
||||||
|
echo "$(date "+%Y-%m-%d %T") ${LOCALDIR} doesn't exist" | tee -a ${LOG}
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
lftp ${HOST} <<- UPLOAD
|
||||||
|
user "${USER}" "${PASS}"
|
||||||
|
cd "${REMOTEDIR}"
|
||||||
|
mput -E "${GLOB}"
|
||||||
|
UPLOAD
|
||||||
|
|
||||||
|
if [ ! $? -eq 0 ]; then
|
||||||
|
echo "$(date "+%Y-%m-%d %T") unable to upload files" | tee -a ${LOG}
|
||||||
|
exit 1
|
||||||
|
fi
|
|
@ -0,0 +1,145 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019-2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.py - Version 2.0 - 20200611
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import csv
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
import urllib.parse
|
||||||
|
import requests
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
CONFIG_FILE = "/home/mischa/netskope/netskope.cnf"
|
||||||
|
if not os.path.isfile(CONFIG_FILE):
|
||||||
|
logging.error(f"The config file {CONFIG_FILE} doesn't exist")
|
||||||
|
sys.exit(1)
|
||||||
|
config = configparser.RawConfigParser()
|
||||||
|
config.read(CONFIG_FILE)
|
||||||
|
NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
|
||||||
|
NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
|
||||||
|
ZS_MAX_DOMAINS = int(config.get('zscaler', 'ZS_MAX_DOMAINS'))
|
||||||
|
ZS_BASE_URI = config.get('zscaler', 'ZS_BASE_URI')
|
||||||
|
ZS_API_KEY = config.get('zscaler', 'ZS_API_KEY')
|
||||||
|
ZS_API_USERNAME = config.get('zscaler', 'ZS_API_USERNAME')
|
||||||
|
ZS_API_PASSWORD = config.get('zscaler', 'ZS_API_PASSWORD')
|
||||||
|
ZS_CATEGORY_NAME = config.get('zscaler', 'ZS_CATEGORY_NAME')
|
||||||
|
ZS_CATEGORY_DESC = config.get('zscaler', 'ZS_CATEGORY_DESC')
|
||||||
|
PROXY = config.get('general', 'PROXY')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
# Use a custom user-agent string
|
||||||
|
UA_STRING = 'Netskope_ZScalerImporter1.0'
|
||||||
|
|
||||||
|
# Set logging.INFO to logging.DEBUG for debug information
|
||||||
|
logging.basicConfig(level=logging.DEBUG)
|
||||||
|
logging = logging.getLogger('Netskope_ZScalerImporter')
|
||||||
|
|
||||||
|
def ntskp_get_domains():
|
||||||
|
ioc_list = []
|
||||||
|
with open('zscaler.txt') as f:
|
||||||
|
ioc_list = f.read().splitlines()
|
||||||
|
logging.debug(ioc_list[:ZS_MAX_DOMAINS])
|
||||||
|
return ioc_list[:ZS_MAX_DOMAINS]
|
||||||
|
|
||||||
|
def zs_auth(headers):
|
||||||
|
# Authenticatie against ZScaler API, fetch and return JSESSIONID
|
||||||
|
now = int(time.time() * 1000)
|
||||||
|
n = str(now)[-6:]
|
||||||
|
r = str(int(n) >> 1).zfill(6)
|
||||||
|
key = ""
|
||||||
|
for i in range(0, len(str(n)), 1):
|
||||||
|
key += ZS_API_KEY[int(str(n)[i])]
|
||||||
|
for j in range(0, len(str(r)), 1):
|
||||||
|
key += ZS_API_KEY[int(str(r)[j])+2]
|
||||||
|
|
||||||
|
uri = f'{ZS_BASE_URI}/authenticatedSession'
|
||||||
|
body = {'apiKey': key,
|
||||||
|
'username': ZS_API_USERNAME,
|
||||||
|
'password': ZS_API_PASSWORD,
|
||||||
|
'timestamp': now}
|
||||||
|
try:
|
||||||
|
r = requests.post(uri, data=json.dumps(body), headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
jsessionid = re.sub(r';.*$', "", r.headers['Set-Cookie'])
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
return jsessionid
|
||||||
|
|
||||||
|
def zs_get_categories(headers):
|
||||||
|
# Find any existing categories matching ZS_CATEGORY_NAME
|
||||||
|
uri = f'{ZS_BASE_URI}/urlCategories/lite'
|
||||||
|
try:
|
||||||
|
r = requests.get(uri, headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
data = r.json()
|
||||||
|
for item in data:
|
||||||
|
if item.get('configuredName') == ZS_CATEGORY_NAME:
|
||||||
|
return item.get('id')
|
||||||
|
return None
|
||||||
|
|
||||||
|
def zs_update_categories(headers, domains, id = None):
|
||||||
|
# Update the ZS_CATEGORY_NAME with blocklist from Netskope
|
||||||
|
description = f'{ZS_CATEGORY_DESC}\n\nLast Updated: {str(time.ctime(int(time.time())))}'
|
||||||
|
body = {'configuredName': ZS_CATEGORY_NAME,
|
||||||
|
'customCategory': 'true',
|
||||||
|
'superCategory': 'SECURITY',
|
||||||
|
'urls': domains,
|
||||||
|
'description': description}
|
||||||
|
try:
|
||||||
|
if id == None:
|
||||||
|
uri = f'{ZS_BASE_URI}/urlCategories'
|
||||||
|
r = requests.post(uri, json=body, headers=headers, proxies=PROXY)
|
||||||
|
else:
|
||||||
|
uri = f'{ZS_BASE_URI}/urlCategories/{str(id)}'
|
||||||
|
r = requests.put(uri, json=body, headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def zs_logout(headers):
|
||||||
|
# Logout from ZScaler
|
||||||
|
uri = f'{ZS_BASE_URI}/authenticatedSession'
|
||||||
|
try:
|
||||||
|
r = requests.delete(uri, headers=headers, proxies=PROXY)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
return None
|
||||||
|
|
||||||
|
##############################################
|
||||||
|
|
||||||
|
request_headers = {'Content-Type': 'application/json', 'Cache-Control': 'no-cache', 'User-Agent': UA_STRING}
|
||||||
|
domains = ntskp_get_domains()
|
||||||
|
request_headers['Cookie'] = zs_auth(request_headers)
|
||||||
|
zs_update_categories(request_headers, domains, zs_get_categories(request_headers))
|
||||||
|
zs_logout(request_headers)
|
||||||
|
logging.info(f'Netskope added {str(len(domains))} domains added to ZScaler custom URL category {ZS_CATEGORY_NAME}')
|
|
@ -0,0 +1,192 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use File::Temp;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $VERBOSE = 1;
|
||||||
|
my $DEBUG = 1;
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $FILENAME = $config->{general}{FILENAME};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %headers = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach( Type => 'text/csv', Path => $FILENAME, Filename => $FILENAME);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
|
||||||
|
unlink $FILENAME;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
|
||||||
|
);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%headers);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %files;
|
||||||
|
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
my $tmp_file = File::Temp->new(UNLINK => 0, TEMPLATE => 'tempXXXXX', DIR => '/tmp');
|
||||||
|
$files{$tmp_file} = $widget->{'name'};
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
open my $fh_out, ">", $tmp_file;
|
||||||
|
print $fh_out $response->{'content'};
|
||||||
|
close $fh_out;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
open my $fh_email_out, ">", $FILENAME;
|
||||||
|
|
||||||
|
for my $csv_file (keys %files) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", $csv_file;
|
||||||
|
my $header = $csv->getline($fh_in);
|
||||||
|
|
||||||
|
print "\n## Widget Name: $files{$csv_file}\nDomains: " if $VERBOSE;
|
||||||
|
print $fh_email_out "$files{$csv_file}\n";
|
||||||
|
|
||||||
|
while (my $row = $csv->getline($fh_in)) {
|
||||||
|
last if ($count == 30);
|
||||||
|
print "$row->[1]," if $VERBOSE;
|
||||||
|
print $fh_email_out "$row->[1],";
|
||||||
|
push @blocklist, $row->[1];
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
print "\n" if $VERBOSE;
|
||||||
|
print $fh_email_out "\n";
|
||||||
|
close $fh_in;
|
||||||
|
unlink $csv_file;
|
||||||
|
}
|
||||||
|
close $fh_email_out;
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%headers, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running..." if $VERBOSE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $VERBOSE;
|
|
@ -0,0 +1,179 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $VERBOSE = 1;
|
||||||
|
my $DEBUG = 0;
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV;
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "SMTP $FROM -> $TO - CSV" if $VERBOSE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'Error', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n",
|
||||||
|
);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "SMTP $FROM -> $TO - ERROR" if $VERBOSE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if $DEBUG;
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my $header = $csv->getline($fh_in);
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $VERBOSE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
while (my $row = $csv->getline($fh_in)) {
|
||||||
|
last if ($count == 30);
|
||||||
|
print "$row->[1]," if $VERBOSE;
|
||||||
|
$EMAIL_CSV .= "$row->[1],";
|
||||||
|
push @blocklist, $row->[1];
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
print "\n" if $VERBOSE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running..." if $VERBOSE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $VERBOSE;
|
|
@ -0,0 +1,183 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "VERBOSE";
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my $header = $csv->getline($fh_in);
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
while (my $row = $csv->getline($fh_in)) {
|
||||||
|
last if ($count == 30);
|
||||||
|
print "$row->[1]," if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$row->[1],";
|
||||||
|
push @blocklist, $row->[1];
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,189 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my $header = $csv->getline($fh_in);
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->[4] < $USER_COUNT) {
|
||||||
|
print "$row->[1]," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->[0] - $row->[1] - $row->[2], $row->[3], $row->[4]\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
$EMAIL_CSV .= "$row->[1],";
|
||||||
|
push @blocklist, $row->[1];
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,190 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "VERBOSE";
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,190 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,199 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
push @blocklist, split (/,/, $row->{'Domain'});
|
||||||
|
} else {
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
}
|
||||||
|
$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
# DEBUG
|
||||||
|
print "$body\n";
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,198 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
push @blocklist, split (/,/, $row->{'Domain'});
|
||||||
|
} else {
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
}
|
||||||
|
$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,207 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
# Version 3.2 - 20200826 - added all fields to CSV export
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Users\n";
|
||||||
|
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
push @blocklist, split (/,/, $row->{'Domain'});
|
||||||
|
$domain = $row->{'Domain'} =~ s/,/ /gr;
|
||||||
|
} else {
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$domain = $row->{'Domain'};
|
||||||
|
|
||||||
|
}
|
||||||
|
#$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
#$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
#zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,205 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my @headers = $csv->column_names($csv->getline($fh_in));
|
||||||
|
print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
|
||||||
|
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
next DOMAIN if ($row->{'Blocked Events'} > 0);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
push @blocklist, split (/,/, $row->{'Domain'});
|
||||||
|
$domain = $row->{'Domain'} =~ s/,/ /gr;
|
||||||
|
} else {
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$domain = $row->{'Domain'};
|
||||||
|
|
||||||
|
}
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
#zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,276 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
my @existing_domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my @headers = $csv->column_names($csv->getline($fh_in));
|
||||||
|
print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
|
||||||
|
|
||||||
|
DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
#next DOMAIN if ($row->{'Blocked Events'} > 0);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
PARSE:
|
||||||
|
for my $item (split (/,/, $row->{'Domain'})) {
|
||||||
|
next PARSE if (grep(/$item/, @existing_domains));
|
||||||
|
push @blocklist, $item;
|
||||||
|
$domain .= $item . " ";
|
||||||
|
}
|
||||||
|
if ($domain) {
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
print "COUNT: $count\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler_get {
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/$id";
|
||||||
|
my $method = "get";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
$json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'urls'};
|
||||||
|
my @convert = ();
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
push @convert, $item;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
return @convert;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub zscaler_push {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @existing_domains = zscaler_get();
|
||||||
|
my @domains = netskope(\@existing_domains);
|
||||||
|
print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
|
||||||
|
zscaler_push(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,277 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
my @existing_domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my @headers = $csv->column_names($csv->getline($fh_in));
|
||||||
|
print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
|
||||||
|
|
||||||
|
DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
#next DOMAIN if ($row->{'Blocked Events'} > 0);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
next DOMAIN if ($row->{'Domain'} =~ "n/a");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
PARSE:
|
||||||
|
for my $item (split (/,/, $row->{'Domain'})) {
|
||||||
|
next PARSE if (grep(/$item/, @existing_domains));
|
||||||
|
push @blocklist, $item;
|
||||||
|
$domain .= $item . " ";
|
||||||
|
}
|
||||||
|
if ($domain) {
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
next DOMAIN if (!grep(/$row->{'Domain'}/, @existing_domains));
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
print "COUNT: $count\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler_get {
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/$id";
|
||||||
|
my $method = "get";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
$json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'urls'};
|
||||||
|
my @convert = ();
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
push @convert, $item;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
return @convert;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub zscaler_push {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @existing_domains = zscaler_get();
|
||||||
|
my @domains = netskope(\@existing_domains);
|
||||||
|
print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
|
||||||
|
zscaler_push(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,280 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
|
||||||
|
# Version 3.3 - 20210121 - filter our entries when domain "n/a"
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
#my $LOGMODE = "";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
my @existing_domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my @headers = $csv->column_names($csv->getline($fh_in));
|
||||||
|
print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
|
||||||
|
|
||||||
|
DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
next DOMAIN if ($row->{'Blocked Events'} > 0);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
next DOMAIN if ($row->{'Domain'} =~ "n/a");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
PARSE:
|
||||||
|
for my $item (split (/,/, $row->{'Domain'})) {
|
||||||
|
next PARSE if (grep(/$item/, @existing_domains));
|
||||||
|
push @blocklist, $item;
|
||||||
|
$domain .= $item . " ";
|
||||||
|
}
|
||||||
|
if ($domain) {
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
print "COUNT: $count\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler_get {
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/$id";
|
||||||
|
my $method = "get";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
$json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'urls'};
|
||||||
|
my @convert = ();
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
push @convert, $item;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
return @convert;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub zscaler_push {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories?action=ADD_TO_LIST";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @existing_domains = zscaler_get();
|
||||||
|
my @domains = netskope(\@existing_domains);
|
||||||
|
print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
|
||||||
|
zscaler_push(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,276 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl
|
||||||
|
# Version 3.0 - 20200615 - rewrite to Perl
|
||||||
|
# Version 3.1 - 20200812 - split domains when comma separated in CSV
|
||||||
|
# Version 3.2 - 20200909 - de-duplication of Zscaler URL category
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.016;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "DEBUG";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV, Filename => 'zscaler_blocklist.csv');
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
my @existing_domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $domain;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
my @headers = $csv->column_names($csv->getline($fh_in));
|
||||||
|
print "*** ", join(" - ", @headers), " ***\n\n" if $LOGMODE;
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
$EMAIL_CSV .= "Application,Domain,Category,CCI,Blocked Events,Users\n";
|
||||||
|
|
||||||
|
DOMAIN: while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
#next DOMAIN if ($row->{'Blocked Events'} > 0);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Blocked Events'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
if ($row->{'Domain'} =~ /,/) {
|
||||||
|
PARSE:
|
||||||
|
for my $item (split (/,/, $row->{'Domain'})) {
|
||||||
|
next PARSE if (grep(/$item/, @existing_domains));
|
||||||
|
push @blocklist, $item;
|
||||||
|
$domain .= $item . " ";
|
||||||
|
}
|
||||||
|
if ($domain) {
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$domain,$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
next DOMAIN if (grep(/$row->{'Domain'}/, @existing_domains));
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$EMAIL_CSV .= "$row->{'Application'},$row->{'Domain'},$row->{'Category'},$row->{'CCI'},$row->{'Blocked Events'},$row->{'Users'}\n";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
print "COUNT: $count\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler_get {
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/$id";
|
||||||
|
my $method = "get";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
$json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'urls'};
|
||||||
|
my @convert = ();
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
push @convert, $item;
|
||||||
|
}
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
return @convert;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sub zscaler_push {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'proxy' => $PROXY, 'cookie_jar' => $jar);
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id?action=ADD_TO_LIST" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
print "$body\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @existing_domains = zscaler_get();
|
||||||
|
my @domains = netskope(\@existing_domains);
|
||||||
|
print "Total Domains Pushed: " . scalar @domains . "\n" if $LOGMODE;
|
||||||
|
zscaler_push(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,351 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
# References:
|
||||||
|
# man curl
|
||||||
|
# https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
|
||||||
|
# https://curl.haxx.se/libcurl/c/easy_getinfo_options.html
|
||||||
|
# http://blog.kenweiner.com/2014/11/http-request-timings-with-curl.html
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import logging
|
||||||
|
import tempfile
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
__version__ = '1.2.1'
|
||||||
|
|
||||||
|
|
||||||
|
PY3 = sys.version_info >= (3,)
|
||||||
|
|
||||||
|
if PY3:
|
||||||
|
xrange = range
|
||||||
|
|
||||||
|
|
||||||
|
# Env class is copied from https://github.com/reorx/getenv/blob/master/getenv.py
|
||||||
|
class Env(object):
|
||||||
|
prefix = 'HTTPSTAT'
|
||||||
|
_instances = []
|
||||||
|
|
||||||
|
def __init__(self, key):
|
||||||
|
self.key = key.format(prefix=self.prefix)
|
||||||
|
Env._instances.append(self)
|
||||||
|
|
||||||
|
def get(self, default=None):
|
||||||
|
return os.environ.get(self.key, default)
|
||||||
|
|
||||||
|
|
||||||
|
ENV_SHOW_BODY = Env('{prefix}_SHOW_BODY')
|
||||||
|
ENV_SHOW_IP = Env('{prefix}_SHOW_IP')
|
||||||
|
ENV_SHOW_SPEED = Env('{prefix}_SHOW_SPEED')
|
||||||
|
ENV_SAVE_BODY = Env('{prefix}_SAVE_BODY')
|
||||||
|
ENV_CURL_BIN = Env('{prefix}_CURL_BIN')
|
||||||
|
ENV_DEBUG = Env('{prefix}_DEBUG')
|
||||||
|
|
||||||
|
|
||||||
|
curl_format = """{
|
||||||
|
"time_namelookup": %{time_namelookup},
|
||||||
|
"time_connect": %{time_connect},
|
||||||
|
"time_appconnect": %{time_appconnect},
|
||||||
|
"time_pretransfer": %{time_pretransfer},
|
||||||
|
"time_redirect": %{time_redirect},
|
||||||
|
"time_starttransfer": %{time_starttransfer},
|
||||||
|
"time_total": %{time_total},
|
||||||
|
"speed_download": %{speed_download},
|
||||||
|
"speed_upload": %{speed_upload},
|
||||||
|
"remote_ip": "%{remote_ip}",
|
||||||
|
"remote_port": "%{remote_port}",
|
||||||
|
"local_ip": "%{local_ip}",
|
||||||
|
"local_port": "%{local_port}"
|
||||||
|
}"""
|
||||||
|
|
||||||
|
https_template = """
|
||||||
|
DNS Lookup TCP Connection TLS Handshake Server Processing Content Transfer
|
||||||
|
[ {a0000} | {a0001} | {a0002} | {a0003} | {a0004} ]
|
||||||
|
| | | | |
|
||||||
|
namelookup:{b0000} | | | |
|
||||||
|
connect:{b0001} | | |
|
||||||
|
pretransfer:{b0002} | |
|
||||||
|
starttransfer:{b0003} |
|
||||||
|
total:{b0004}
|
||||||
|
"""[1:]
|
||||||
|
|
||||||
|
http_template = """
|
||||||
|
DNS Lookup TCP Connection Server Processing Content Transfer
|
||||||
|
[ {a0000} | {a0001} | {a0003} | {a0004} ]
|
||||||
|
| | | |
|
||||||
|
namelookup:{b0000} | | |
|
||||||
|
connect:{b0001} | |
|
||||||
|
starttransfer:{b0003} |
|
||||||
|
total:{b0004}
|
||||||
|
"""[1:]
|
||||||
|
|
||||||
|
|
||||||
|
# Color code is copied from https://github.com/reorx/python-terminal-color/blob/master/color_simple.py
|
||||||
|
ISATTY = sys.stdout.isatty()
|
||||||
|
|
||||||
|
|
||||||
|
def make_color(code):
|
||||||
|
def color_func(s):
|
||||||
|
if not ISATTY:
|
||||||
|
return s
|
||||||
|
tpl = '\x1b[{}m{}\x1b[0m'
|
||||||
|
return tpl.format(code, s)
|
||||||
|
return color_func
|
||||||
|
|
||||||
|
|
||||||
|
red = make_color(31)
|
||||||
|
green = make_color(32)
|
||||||
|
yellow = make_color(33)
|
||||||
|
blue = make_color(34)
|
||||||
|
magenta = make_color(35)
|
||||||
|
cyan = make_color(36)
|
||||||
|
|
||||||
|
bold = make_color(1)
|
||||||
|
underline = make_color(4)
|
||||||
|
|
||||||
|
grayscale = {(i - 232): make_color('38;5;' + str(i)) for i in xrange(232, 256)}
|
||||||
|
|
||||||
|
|
||||||
|
def quit(s, code=0):
|
||||||
|
if s is not None:
|
||||||
|
print(s)
|
||||||
|
sys.exit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def print_help():
|
||||||
|
help = """
|
||||||
|
Usage: httpstat URL [CURL_OPTIONS]
|
||||||
|
httpstat -h | --help
|
||||||
|
httpstat --version
|
||||||
|
|
||||||
|
Arguments:
|
||||||
|
URL url to request, could be with or without `http(s)://` prefix
|
||||||
|
|
||||||
|
Options:
|
||||||
|
CURL_OPTIONS any curl supported options, except for -w -D -o -S -s,
|
||||||
|
which are already used internally.
|
||||||
|
-h --help show this screen.
|
||||||
|
--version show version.
|
||||||
|
|
||||||
|
Environments:
|
||||||
|
HTTPSTAT_SHOW_BODY Set to `true` to show response body in the output,
|
||||||
|
note that body length is limited to 1023 bytes, will be
|
||||||
|
truncated if exceeds. Default is `false`.
|
||||||
|
HTTPSTAT_SHOW_IP By default httpstat shows remote and local IP/port address.
|
||||||
|
Set to `false` to disable this feature. Default is `true`.
|
||||||
|
HTTPSTAT_SHOW_SPEED Set to `true` to show download and upload speed.
|
||||||
|
Default is `false`.
|
||||||
|
HTTPSTAT_SAVE_BODY By default httpstat stores body in a tmp file,
|
||||||
|
set to `false` to disable this feature. Default is `true`
|
||||||
|
HTTPSTAT_CURL_BIN Indicate the curl bin path to use. Default is `curl`
|
||||||
|
from current shell $PATH.
|
||||||
|
HTTPSTAT_DEBUG Set to `true` to see debugging logs. Default is `false`
|
||||||
|
"""[1:-1]
|
||||||
|
print(help)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = sys.argv[1:]
|
||||||
|
if not args:
|
||||||
|
print_help()
|
||||||
|
quit(None, 0)
|
||||||
|
|
||||||
|
# get envs
|
||||||
|
show_body = 'true' in ENV_SHOW_BODY.get('false').lower()
|
||||||
|
show_ip = 'true' in ENV_SHOW_IP.get('true').lower()
|
||||||
|
show_speed = 'true'in ENV_SHOW_SPEED.get('false').lower()
|
||||||
|
save_body = 'true' in ENV_SAVE_BODY.get('true').lower()
|
||||||
|
curl_bin = ENV_CURL_BIN.get('curl')
|
||||||
|
is_debug = 'true' in ENV_DEBUG.get('false').lower()
|
||||||
|
|
||||||
|
# configure logging
|
||||||
|
if is_debug:
|
||||||
|
log_level = logging.DEBUG
|
||||||
|
else:
|
||||||
|
log_level = logging.INFO
|
||||||
|
logging.basicConfig(level=log_level)
|
||||||
|
lg = logging.getLogger('httpstat')
|
||||||
|
|
||||||
|
# log envs
|
||||||
|
lg.debug('Envs:\n%s', '\n'.join(' {}={}'.format(i.key, i.get('')) for i in Env._instances))
|
||||||
|
lg.debug('Flags: %s', dict(
|
||||||
|
show_body=show_body,
|
||||||
|
show_ip=show_ip,
|
||||||
|
show_speed=show_speed,
|
||||||
|
save_body=save_body,
|
||||||
|
curl_bin=curl_bin,
|
||||||
|
is_debug=is_debug,
|
||||||
|
))
|
||||||
|
|
||||||
|
# get url
|
||||||
|
url = args[0]
|
||||||
|
if url in ['-h', '--help']:
|
||||||
|
print_help()
|
||||||
|
quit(None, 0)
|
||||||
|
elif url == '--version':
|
||||||
|
print('httpstat {}'.format(__version__))
|
||||||
|
quit(None, 0)
|
||||||
|
|
||||||
|
curl_args = args[1:]
|
||||||
|
|
||||||
|
# check curl args
|
||||||
|
exclude_options = [
|
||||||
|
'-w', '--write-out',
|
||||||
|
'-D', '--dump-header',
|
||||||
|
'-o', '--output',
|
||||||
|
'-s', '--silent',
|
||||||
|
]
|
||||||
|
for i in exclude_options:
|
||||||
|
if i in curl_args:
|
||||||
|
quit(yellow('Error: {} is not allowed in extra curl args'.format(i)), 1)
|
||||||
|
|
||||||
|
# tempfile for output
|
||||||
|
bodyf = tempfile.NamedTemporaryFile(delete=False)
|
||||||
|
bodyf.close()
|
||||||
|
|
||||||
|
headerf = tempfile.NamedTemporaryFile(delete=False)
|
||||||
|
headerf.close()
|
||||||
|
|
||||||
|
# run cmd
|
||||||
|
cmd_env = os.environ.copy()
|
||||||
|
cmd_env.update(
|
||||||
|
LC_ALL='C',
|
||||||
|
)
|
||||||
|
cmd_core = [curl_bin, '-w', curl_format, '-D', headerf.name, '-o', bodyf.name, '-s', '-S']
|
||||||
|
cmd = cmd_core + curl_args + [url]
|
||||||
|
lg.debug('cmd: %s', cmd)
|
||||||
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=cmd_env)
|
||||||
|
out, err = p.communicate()
|
||||||
|
if PY3:
|
||||||
|
out, err = out.decode(), err.decode()
|
||||||
|
lg.debug('out: %s', out)
|
||||||
|
|
||||||
|
# print stderr
|
||||||
|
if p.returncode == 0:
|
||||||
|
if err:
|
||||||
|
print(grayscale[16](err))
|
||||||
|
else:
|
||||||
|
_cmd = list(cmd)
|
||||||
|
_cmd[2] = '<output-format>'
|
||||||
|
_cmd[4] = '<tempfile>'
|
||||||
|
_cmd[6] = '<tempfile>'
|
||||||
|
print('> {}'.format(' '.join(_cmd)))
|
||||||
|
quit(yellow('curl error: {}'.format(err)), p.returncode)
|
||||||
|
|
||||||
|
# parse output
|
||||||
|
try:
|
||||||
|
d = json.loads(out)
|
||||||
|
except ValueError as e:
|
||||||
|
print(yellow('Could not decode json: {}'.format(e)))
|
||||||
|
print('curl result:', p.returncode, grayscale[16](out), grayscale[16](err))
|
||||||
|
quit(None, 1)
|
||||||
|
for k in d:
|
||||||
|
if k.startswith('time_'):
|
||||||
|
d[k] = int(d[k] * 1000)
|
||||||
|
|
||||||
|
# calculate ranges
|
||||||
|
d.update(
|
||||||
|
range_dns=d['time_namelookup'],
|
||||||
|
range_connection=d['time_connect'] - d['time_namelookup'],
|
||||||
|
range_ssl=d['time_pretransfer'] - d['time_connect'],
|
||||||
|
range_server=d['time_starttransfer'] - d['time_pretransfer'],
|
||||||
|
range_transfer=d['time_total'] - d['time_starttransfer'],
|
||||||
|
)
|
||||||
|
|
||||||
|
# ip
|
||||||
|
if show_ip:
|
||||||
|
s = 'Connected to {}:{} from {}:{}'.format(
|
||||||
|
cyan(d['remote_ip']), cyan(d['remote_port']),
|
||||||
|
d['local_ip'], d['local_port'],
|
||||||
|
)
|
||||||
|
print(s)
|
||||||
|
print()
|
||||||
|
|
||||||
|
# print header & body summary
|
||||||
|
with open(headerf.name, 'r') as f:
|
||||||
|
headers = f.read().strip()
|
||||||
|
# remove header file
|
||||||
|
lg.debug('rm header file %s', headerf.name)
|
||||||
|
os.remove(headerf.name)
|
||||||
|
|
||||||
|
for loop, line in enumerate(headers.split('\n')):
|
||||||
|
if loop == 0:
|
||||||
|
p1, p2 = tuple(line.split('/'))
|
||||||
|
print(green(p1) + grayscale[14]('/') + cyan(p2))
|
||||||
|
else:
|
||||||
|
pos = line.find(':')
|
||||||
|
print(grayscale[14](line[:pos + 1]) + cyan(line[pos + 1:]))
|
||||||
|
|
||||||
|
print()
|
||||||
|
|
||||||
|
# body
|
||||||
|
if show_body:
|
||||||
|
body_limit = 1024
|
||||||
|
with open(bodyf.name, 'r') as f:
|
||||||
|
body = f.read().strip()
|
||||||
|
body_len = len(body)
|
||||||
|
|
||||||
|
if body_len > body_limit:
|
||||||
|
print(body[:body_limit] + cyan('...'))
|
||||||
|
print()
|
||||||
|
s = '{} is truncated ({} out of {})'.format(green('Body'), body_limit, body_len)
|
||||||
|
if save_body:
|
||||||
|
s += ', stored in: {}'.format(bodyf.name)
|
||||||
|
print(s)
|
||||||
|
else:
|
||||||
|
print(body)
|
||||||
|
else:
|
||||||
|
if save_body:
|
||||||
|
print('{} stored in: {}'.format(green('Body'), bodyf.name))
|
||||||
|
|
||||||
|
# remove body file
|
||||||
|
if not save_body:
|
||||||
|
lg.debug('rm body file %s', bodyf.name)
|
||||||
|
os.remove(bodyf.name)
|
||||||
|
|
||||||
|
# print stat
|
||||||
|
if url.startswith('https://'):
|
||||||
|
template = https_template
|
||||||
|
else:
|
||||||
|
template = http_template
|
||||||
|
|
||||||
|
# colorize template first line
|
||||||
|
tpl_parts = template.split('\n')
|
||||||
|
tpl_parts[0] = grayscale[16](tpl_parts[0])
|
||||||
|
template = '\n'.join(tpl_parts)
|
||||||
|
|
||||||
|
def fmta(s):
|
||||||
|
return cyan('{:^7}'.format(str(s) + 'ms'))
|
||||||
|
|
||||||
|
def fmtb(s):
|
||||||
|
return cyan('{:<7}'.format(str(s) + 'ms'))
|
||||||
|
|
||||||
|
stat = template.format(
|
||||||
|
# a
|
||||||
|
a0000=fmta(d['range_dns']),
|
||||||
|
a0001=fmta(d['range_connection']),
|
||||||
|
a0002=fmta(d['range_ssl']),
|
||||||
|
a0003=fmta(d['range_server']),
|
||||||
|
a0004=fmta(d['range_transfer']),
|
||||||
|
# b
|
||||||
|
b0000=fmtb(d['time_namelookup']),
|
||||||
|
b0001=fmtb(d['time_connect']),
|
||||||
|
b0002=fmtb(d['time_pretransfer']),
|
||||||
|
b0003=fmtb(d['time_starttransfer']),
|
||||||
|
b0004=fmtb(d['time_total']),
|
||||||
|
)
|
||||||
|
print()
|
||||||
|
print(stat)
|
||||||
|
|
||||||
|
# speed, originally bytes per second
|
||||||
|
if show_speed:
|
||||||
|
print('speed_download: {:.1f} KiB/s, speed_upload: {:.1f} KiB/s'.format(
|
||||||
|
d['speed_download'] / 1024, d['speed_upload'] / 1024))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -0,0 +1,29 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
from operator import itemgetter
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="API Call to collect data")
|
||||||
|
parser.add_argument("tenant", type=str, help="Tenant Name")
|
||||||
|
parser.add_argument("token", type=str, help="Tenat API Token")
|
||||||
|
parser.add_argument("-t", "--timeperiod", type=int, default='604800', help="Timeperiod (default: 604800)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
tenant = args.tenant
|
||||||
|
token = args.token
|
||||||
|
timeperiod = args.timeperiod
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
base_url = "https://{}.goskope.com/api/v1/events?token={}&type=page&timeperiod={}".format(tenant, token, timeperiod)
|
||||||
|
|
||||||
|
req = urllib.request.Request(base_url)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
content = response.read()
|
||||||
|
json_content = json.loads(content)
|
||||||
|
print(json.dumps(json_content, indent=4, sort_keys=True))
|
|
@ -0,0 +1,46 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2019, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Version 1.0 - 20191028
|
||||||
|
#
|
||||||
|
# Measure timing for DNS lookup as well as HTTP page load
|
||||||
|
#
|
||||||
|
# Requires:
|
||||||
|
# - Python 3.x
|
||||||
|
#
|
||||||
|
import argparse
|
||||||
|
import socket
|
||||||
|
import time
|
||||||
|
import ssl
|
||||||
|
import urllib.request
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="Measure load times", epilog="2019 (c) Netskope")
|
||||||
|
parser.add_argument("url", type=str, help="url (eg. https://google.com)")
|
||||||
|
|
||||||
|
try:
|
||||||
|
args = parser.parse_args()
|
||||||
|
url = args.url
|
||||||
|
|
||||||
|
except argparse.ArgumentError as e:
|
||||||
|
print(str(e))
|
||||||
|
|
||||||
|
print (url, "timing:")
|
||||||
|
|
||||||
|
urlinfo = urlparse(url)
|
||||||
|
request_headers = {'Cache-Control': 'no-cache', 'User-Agent': 'Mozilla/5.0'}
|
||||||
|
no_cert_check = ssl.create_default_context()
|
||||||
|
no_cert_check.check_hostname=False
|
||||||
|
no_cert_check.verify_mode=ssl.CERT_NONE
|
||||||
|
|
||||||
|
start = time.time()
|
||||||
|
ip = socket.gethostbyname(urlinfo.netloc)
|
||||||
|
dns_time = time.time()-start
|
||||||
|
print ("DNS Lookup:\t{:.3f} seconds".format(dns_time))
|
||||||
|
|
||||||
|
start = time.time()
|
||||||
|
req = urllib.request.Request(url, headers=request_headers)
|
||||||
|
content = urllib.request.urlopen(req, context=no_cert_check).read()
|
||||||
|
load_time = time.time()-start
|
||||||
|
print ("Page Load:\t{:.3f} seconds".format(load_time))
|
||||||
|
print ("w/o DNS Lookup:\t{:.3f} seconds".format(load_time-dns_time))
|
|
@ -0,0 +1,24 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use Cpanel::JSON::XS;
|
||||||
|
|
||||||
|
my $NTSKP_TENANT = "https://oss.de.goskope.com";
|
||||||
|
my $NTSKP_TOKEN = "0cfe04c4237cc33dc7f383af5ddbe2e3";
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=86400&groupby=application&query=access_method+eq+Client+and+action+eq+block";
|
||||||
|
#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'";
|
||||||
|
#my $uri = "$NTSKP_TENANT/api/v1/report?token=$NTSKP_TOKEN&timeperiod=86400&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'";
|
||||||
|
my $response = HTTP::Tiny->new->get($uri);
|
||||||
|
my $json = Cpanel::JSON::XS->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'};
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
if (exists($item->{'app'})) {
|
||||||
|
print ".";
|
||||||
|
#print $item->{'app'} . ", ";
|
||||||
|
#say $item->{'sessions'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
say "";
|
|
@ -0,0 +1,45 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use Cpanel::JSON::XS;
|
||||||
|
|
||||||
|
my $file;
|
||||||
|
{
|
||||||
|
local $/;
|
||||||
|
open my $fh, "<", "amsjson.txt";
|
||||||
|
$file = <$fh>;
|
||||||
|
close $fh;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $json = Cpanel::JSON::XS->new->utf8->decode($file);
|
||||||
|
my $data = $json->{'data'};
|
||||||
|
my $domain;
|
||||||
|
my $cci;
|
||||||
|
|
||||||
|
for (my $i = 0; $i < (@{$data}); $i++) {
|
||||||
|
#print "Timestamp: $data->[$i]->{'timestamp'}\n";
|
||||||
|
#print "Domain: $data->[$i]->{'domain'}\n";
|
||||||
|
|
||||||
|
if (!$data->[$i]->{'domain'}) {
|
||||||
|
my $url = $data->[$i]->{'url'};
|
||||||
|
$url =~ s!^https?://(?:www\.)?!!i;
|
||||||
|
$url =~ s!/.*!!;
|
||||||
|
$url =~ s/[\?\#\:].*//;
|
||||||
|
$domain = $url;
|
||||||
|
} else {
|
||||||
|
$domain = $data->[$i]->{'domain'};
|
||||||
|
}
|
||||||
|
if ($data->[$i]->{'cci'}) {
|
||||||
|
$cci = $data->[$i]->{'cci'};
|
||||||
|
} else {
|
||||||
|
$cci = 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
#print "Category: $data->[$i]->{'category'}\n";
|
||||||
|
#print "CCI: $data->[$i]->{'ccl'}\n";
|
||||||
|
#print "User: $data->[$i]->{'user'}\n";
|
||||||
|
my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
|
||||||
|
print "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'}\n";
|
||||||
|
}
|
|
@ -0,0 +1,69 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use Config::Tiny;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use Cpanel::JSON::XS;
|
||||||
|
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
|
||||||
|
my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
|
||||||
|
my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
|
||||||
|
|
||||||
|
my $uri;
|
||||||
|
my $skip = 0;
|
||||||
|
my $response;
|
||||||
|
my $json;
|
||||||
|
my $data;
|
||||||
|
my $length;
|
||||||
|
my $domain;
|
||||||
|
my $cci;
|
||||||
|
|
||||||
|
my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt";
|
||||||
|
print "File: $file_out\n";
|
||||||
|
print "Tenant: $NTSKP_TENANT\n";
|
||||||
|
|
||||||
|
while ($skip < 500000) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip";
|
||||||
|
$response = HTTP::Tiny->new->get($uri);
|
||||||
|
print "HTTP: $response->{status} $response->{reason}\n";
|
||||||
|
$json = Cpanel::JSON::XS->new->utf8->decode($response->{content});
|
||||||
|
print "API: $json->{'status'}\n";
|
||||||
|
$data = $json->{'data'};
|
||||||
|
|
||||||
|
$length = (@{$data});
|
||||||
|
if ($length == 0) {
|
||||||
|
print "All data collected\n";
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
|
||||||
|
open my $fh_out, ">>", $file_out;
|
||||||
|
for (my $i = 0; $i < $length; $i++) {
|
||||||
|
if (!$data->[$i]->{'domain'}) {
|
||||||
|
my $url = $data->[$i]->{'url'};
|
||||||
|
$url =~ s!^https?://(?:www\.)?!!i;
|
||||||
|
$url =~ s!/.*!!;
|
||||||
|
$url =~ s/[\?\#\:].*//;
|
||||||
|
$domain = $url;
|
||||||
|
} else {
|
||||||
|
$domain = $data->[$i]->{'domain'};
|
||||||
|
}
|
||||||
|
if ($data->[$i]->{'cci'}) {
|
||||||
|
$cci = $data->[$i]->{'cci'};
|
||||||
|
} else {
|
||||||
|
$cci = 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
|
||||||
|
print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n";
|
||||||
|
}
|
||||||
|
close $fh_out;
|
||||||
|
$skip += 5000;
|
||||||
|
#print "Next batch $skip\n";
|
||||||
|
}
|
||||||
|
print "Done\n";
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
#use Cpanel::JSON::XS;
|
||||||
|
use JSON::PP;
|
||||||
|
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
|
||||||
|
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD";
|
||||||
|
my $response = HTTP::Tiny->new->get($uri);
|
||||||
|
#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
|
||||||
|
my $json = JSON::PP->new->pretty(1)->encode($response->{content});
|
||||||
|
print $json;
|
|
@ -0,0 +1,21 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
#use Cpanel::JSON::XS;
|
||||||
|
use JSON::PP;
|
||||||
|
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}->{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}->{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_PERIOD = $config->{netskope}->{NTSKP_PERIOD};
|
||||||
|
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
|
||||||
|
my $response = HTTP::Tiny->new->get($uri);
|
||||||
|
#my $json = Cpanel::JSON::XS->new->indent(1)->encode($response->{content});
|
||||||
|
my $json = JSON::PP->new->pretty(1)->encode($response->{content});
|
||||||
|
print $json;
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use Config::Tiny;
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use JSON::PP;
|
||||||
|
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
|
||||||
|
my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
|
||||||
|
my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
|
||||||
|
my $from_email = 'mischa@netskope.com';
|
||||||
|
my $to_email = 'mischa@netskope.com';
|
||||||
|
my $subject = "AZ Blocklist Report";
|
||||||
|
|
||||||
|
#print "Tenant: $NTSKP_TENANT\n";
|
||||||
|
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=498";
|
||||||
|
my $response = HTTP::Tiny->new->get($uri);
|
||||||
|
#print "HTTP: $response->{status} $response->{reason}\n";
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{content});
|
||||||
|
#print "API: $json->{'status'}\n";
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
|
||||||
|
my $length = (@{$data});
|
||||||
|
if ($length == 0) {
|
||||||
|
print "No widgets found\n";
|
||||||
|
last;
|
||||||
|
}
|
||||||
|
|
||||||
|
open my $fh_email, "|-", "/usr/sbin/sendmail -t";
|
||||||
|
printf $fh_email "To: %s\n", $to_email;
|
||||||
|
printf $fh_email "From: %s\n", $from_email;
|
||||||
|
printf $fh_email "Subject: %s\n\n", $subject;
|
||||||
|
|
||||||
|
for (my $i = 0; $i < $length; $i++) {
|
||||||
|
print "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
|
||||||
|
print $fh_email "$data->[$i]->{'id'} - $data->[$i]->{'name'}\n";
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$data->[$i]->{'id'}";
|
||||||
|
$response = HTTP::Tiny->new->get($uri);
|
||||||
|
#print "HTTP: $response->{status} $response->{reason}\n";
|
||||||
|
my $count = 0;
|
||||||
|
foreach (split(/\r\n/, $response->{content})) {
|
||||||
|
last if ($count == 30);
|
||||||
|
my @fields = split(/,/);
|
||||||
|
next if ($fields[1] =~ '"');
|
||||||
|
print "$fields[1],";
|
||||||
|
print $fh_email "$fields[1],";
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
print "\n";
|
||||||
|
print $fh_email "\n";
|
||||||
|
}
|
||||||
|
close $fh_email;
|
|
@ -0,0 +1,20 @@
|
||||||
|
#!/usr/bin/perl -w
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use File::Temp qw/ tempfile tempdir /;
|
||||||
|
use Text::CSV;
|
||||||
|
|
||||||
|
my $file = "widget-18465-20200611.txt";
|
||||||
|
open my $fh, "<", $file;
|
||||||
|
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
|
||||||
|
my $header = $csv->getline($fh);
|
||||||
|
|
||||||
|
while (my $row = $csv->getline($fh)) {
|
||||||
|
print "$row->[1]\n";
|
||||||
|
}
|
||||||
|
close $fh;
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
<html><head>
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
|
||||||
|
</head>
|
||||||
|
<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
|
||||||
|
<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
|
||||||
|
Unfortunately, the service you are using has been targeted by a
|
||||||
|
very sophisticated password attack. To protect you, the security
|
||||||
|
team recommended that we reset all customer passwords immediately.
|
||||||
|
Effective immediately, you will be required to reset your service
|
||||||
|
password before you can login again. To reset your password please
|
||||||
|
use your regular service password reset link.
|
||||||
|
<div class=""><br class=""></div>
|
||||||
|
<div class="">
|
||||||
|
<a href="https://docs.google.com/forms/d/e/1FAIpQLSeuq5BnjifExxv7hmY5yaC0xBDyAk1IlEbxCEnqhG72brnGmQ/viewform?fbzx=-5005650978102270379" class="">Password Reset</a>
|
||||||
|
<br class="">
|
||||||
|
<div class=""><br class=""></div></div>
|
||||||
|
<div class="">Regards,</div>
|
||||||
|
<div class=""><br class=""></div>
|
||||||
|
<div class="">Your friendly neighbourhood scammer</div>
|
||||||
|
<div class=""><br class=""></div></div>
|
||||||
|
</body></html>
|
|
@ -0,0 +1,12 @@
|
||||||
|
# high5.nl
|
||||||
|
#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
|
||||||
|
#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@high5.nl
|
||||||
|
# ntskp.com
|
||||||
|
cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
|
||||||
|
cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" herman.akker23@ntskp.com
|
||||||
|
# Microsoft
|
||||||
|
cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
|
||||||
|
cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
|
||||||
|
# Died
|
||||||
|
#cat /home/mischa/ntskp-spam.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
|
||||||
|
#cat /home/mischa/ntskp-phishing.txt | mail -s "$(echo -e "Important\nFrom: Scammer <scammer@local>\nContent-Type: text/html")" mischa@M365x857260.onmicrosoft.com
|
|
@ -0,0 +1,17 @@
|
||||||
|
<html><head>
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii">
|
||||||
|
</head>
|
||||||
|
<body style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
|
||||||
|
<meta http-equiv="Content-Type" content="text/html; charset=us-ascii" class="">
|
||||||
|
<div style="word-wrap: break-word; -webkit-nbsp-mode: space; line-break: after-white-space;" class="">
|
||||||
|
Please have a look at this CV at your earliest convenience.
|
||||||
|
<div class=""><br class=""></div>
|
||||||
|
<div class="">
|
||||||
|
<a href="http://jn.gs/I7a" class="">LinkedIn CV</a>
|
||||||
|
<br class="">
|
||||||
|
<div class=""><br class=""></div></div>
|
||||||
|
<div class="">Regards,</div>
|
||||||
|
<div class=""><br class=""></div>
|
||||||
|
<div class="">Your friendly neighbourhood scammer</div>
|
||||||
|
<div class=""><br class=""></div></div>
|
||||||
|
</body></html>
|
|
@ -0,0 +1,35 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
# Look for oss.cnf file in current working directory
|
||||||
|
CONFIG_FILE = "./oss.cnf"
|
||||||
|
if not os.path.isfile(CONFIG_FILE):
|
||||||
|
logging.error(f"The config file {CONFIG_FILE} doesn't exist")
|
||||||
|
sys.exit(1)
|
||||||
|
config = configparser.RawConfigParser()
|
||||||
|
config.read(CONFIG_FILE)
|
||||||
|
NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
|
||||||
|
NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
|
||||||
|
NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
ssl_session = requests.Session()
|
||||||
|
uri = f"{NTSKP_TENANT}/api/v1/alerts?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&groupby=application&query=access_method+eq+Client+and+action+eq+block"
|
||||||
|
try:
|
||||||
|
r = ssl_session.get(uri)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
json = r.json()
|
||||||
|
if 'data' in json:
|
||||||
|
for item in json['data']:
|
||||||
|
if 'app' in item:
|
||||||
|
print(f"{item['app']} - {item['category']}", end=', ')
|
||||||
|
print()
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
# Look for oss.cnf file in current working directory
|
||||||
|
CONFIG_FILE = "./oss.cnf"
|
||||||
|
if not os.path.isfile(CONFIG_FILE):
|
||||||
|
logging.error(f"The config file {CONFIG_FILE} doesn't exist")
|
||||||
|
sys.exit(1)
|
||||||
|
config = configparser.RawConfigParser()
|
||||||
|
config.read(CONFIG_FILE)
|
||||||
|
NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
|
||||||
|
NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
|
||||||
|
NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
ssl_session = requests.Session()
|
||||||
|
uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Under_Review'"
|
||||||
|
try:
|
||||||
|
r = ssl_session.get(uri)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
json = r.json()
|
||||||
|
if 'data' in json:
|
||||||
|
for item in json['data']:
|
||||||
|
print(f"{item['app']}", end=', ')
|
||||||
|
print()
|
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import requests
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
# Look for oss.cnf file in current working directory
|
||||||
|
CONFIG_FILE = "./oss.cnf"
|
||||||
|
if not os.path.isfile(CONFIG_FILE):
|
||||||
|
logging.error(f"The config file {CONFIG_FILE} doesn't exist")
|
||||||
|
sys.exit(1)
|
||||||
|
config = configparser.RawConfigParser()
|
||||||
|
config.read(CONFIG_FILE)
|
||||||
|
NTSKP_TENANT = config.get('netskope', 'NTSKP_TENANT')
|
||||||
|
NTSKP_TOKEN = config.get('netskope', 'NTSKP_TOKEN')
|
||||||
|
NTSKP_PERIOD = config.get('netskope', 'NTSKP_PERIOD')
|
||||||
|
|
||||||
|
###############################################
|
||||||
|
|
||||||
|
ssl_session = requests.Session()
|
||||||
|
uri = f"{NTSKP_TENANT}/api/v1/report?token={NTSKP_TOKEN}&timeperiod={NTSKP_PERIOD}&type=connection&groupby=application&query=app-cci-app-tag+eq+'Pending_GRC_Review'"
|
||||||
|
try:
|
||||||
|
r = ssl_session.get(uri)
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f'Error: {str(e)}')
|
||||||
|
sys.exit(1)
|
||||||
|
json = r.json()
|
||||||
|
if 'data' in json:
|
||||||
|
for item in json['data']:
|
||||||
|
print(f"{item['app']}", end=', ')
|
||||||
|
print()
|
|
@ -0,0 +1,107 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "";
|
||||||
|
#my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./tbi.cnf');
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_TIMEPERIOD = $config->{netskope}{NTSKP_TIMEPERIOD};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/alerts?token=$NTSKP_TOKEN&timeperiod=$NTSKP_TIMEPERIOD&type=policy";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'};
|
||||||
|
printf "%-7s %-45s %-35s %s\n", "Action", "Page", "Policy", "Category";
|
||||||
|
say "#############################################################################################################################";
|
||||||
|
|
||||||
|
my @seen;
|
||||||
|
for my $item (@{$data}) {
|
||||||
|
if (exists($item->{'page'})) {
|
||||||
|
next if (grep {$_ eq $item->{'site'}} @seen);
|
||||||
|
printf "%-7s %-45s %-35s %s\n", $item->{'action'}, $item->{'page'}, $item->{'policy'}, $item->{'category'};
|
||||||
|
push @seen, $item->{'site'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
netskope();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,199 @@
|
||||||
|
#!/usr/bin/env perl
|
||||||
|
#
|
||||||
|
# Copyright 2020, Mischa Peters <mischa AT netskope DOT com>, Netskope.
|
||||||
|
# Netskope_ZScalerImporter.pl - Version 3.0 - 20200615
|
||||||
|
#
|
||||||
|
# Permission to use, copy, modify, and distribute this software for any
|
||||||
|
# purpose with or without fee is hereby granted, provided that the above
|
||||||
|
# copyright notice and this permission notice appear in all copies.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||||
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||||
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||||
|
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||||
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||||
|
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||||
|
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
#
|
||||||
|
# ZScaler integration with Netskope
|
||||||
|
#
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
use MIME::Lite;
|
||||||
|
|
||||||
|
my $LOGMODE = "";
|
||||||
|
my @CONFIG_FILES = grep { -e } ('./netskope.cnf', './.netskope.cnf', '/etc/netskope.cnf', "$ENV{'HOME'}/.netskope.cnf", "$ENV{'HOME'}/netskope.cnf");
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILES[-1], 'utf8');
|
||||||
|
my $USER_COUNT = $config->{report}{USER_COUNT};
|
||||||
|
my $MAX_DOMAIN= $config->{report}{MAX_DOMAIN};
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
my $PROXY = $config->{general}{PROXY};
|
||||||
|
my $SMTP = $config->{general}{SMTP};
|
||||||
|
my $FROM = $config->{general}{FROM};
|
||||||
|
my $TO = $config->{general}{TO};
|
||||||
|
my $SUBJECT = $config->{general}{SUBJECT};
|
||||||
|
my $TEXT = $config->{general}{TEXT} . "\n\n";
|
||||||
|
my %HEADERS = ("Content-Type" => "application/json", "Cache-Control" => "no-cache");
|
||||||
|
my $EMAIL_CSV = "";
|
||||||
|
|
||||||
|
### Netskope ###
|
||||||
|
sub mail_csv {
|
||||||
|
my $msg = MIME::Lite->new(From=> $FROM, To => $TO, Subject => $SUBJECT, Type => 'TEXT', Data => $TEXT);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Attach CSV" if $LOGMODE;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub _check_return {
|
||||||
|
my ($status, $content, $uri) = @_;
|
||||||
|
if ($status =~ /^2/ && $LOGMODE) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n";
|
||||||
|
print "CONTENT:\n$content\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
}
|
||||||
|
if ($status !~ /^2/) {
|
||||||
|
print "URI: $uri\nHTTP RESPONSE: $status\n$content\n";
|
||||||
|
my $msg = MIME::Lite->new(From => $FROM, To => $TO, Subject => 'API Error along the way', Type => 'TEXT',
|
||||||
|
Data => "URI: $uri\nHTTP RESPONSE: $status\n$content\n\n",
|
||||||
|
);
|
||||||
|
$msg->attach(Type => 'text/csv', Data => $EMAIL_CSV);
|
||||||
|
$msg->send('smtp', $SMTP, Debug=>0);
|
||||||
|
say "MAIL From: $FROM -> $TO - Error Notification" if $LOGMODE;
|
||||||
|
say "exit 1";
|
||||||
|
exit 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sub netskope {
|
||||||
|
### Collect widget IDs
|
||||||
|
my $uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=reportInfo&id=$NTSKP_REPORTID";
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
my $response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $data = $json->{'data'}->{'latestScheduledRunInfo'}->{'widgets'};
|
||||||
|
if (!$data) { _check_return(404, $response->{'content'}, "No Widget Data"); }
|
||||||
|
my %csv_content;
|
||||||
|
|
||||||
|
### Collect widget data and write to CSV
|
||||||
|
for my $widget (@{$data}) {
|
||||||
|
$uri = "$NTSKP_TENANT/api/v1/reports?token=$NTSKP_TOKEN&op=widgetData&id=$widget->{'id'}";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
print "\n#DEBUG#\nWidget Name: " . $widget->{'name'} . "\nWidgetID: " . $widget->{'id'} . " (ReportID: $NTSKP_REPORTID)\n" . $response->{'content'} if ($LOGMODE && $LOGMODE eq "DEBUG");
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
$csv_content{$widget->{'name'}} = $response->{'content'};
|
||||||
|
}
|
||||||
|
|
||||||
|
### Process domains from CSV
|
||||||
|
my @blocklist;
|
||||||
|
for my $widget_name (keys %csv_content) {
|
||||||
|
my $count = 0;
|
||||||
|
my $csv = Text::CSV->new({binary => 1, auto_diag => 1});
|
||||||
|
open my $fh_in, "<", \$csv_content{$widget_name};
|
||||||
|
$csv->column_names($csv->getline($fh_in));
|
||||||
|
# "Application","Domain","Category","CCI","Users"
|
||||||
|
|
||||||
|
print "\n## Widget Name: $widget_name\n## Domains: " if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "$widget_name\n";
|
||||||
|
DOMAIN:
|
||||||
|
while (my $row = $csv->getline_hr($fh_in)) {
|
||||||
|
last DOMAIN if ($count == $MAX_DOMAIN);
|
||||||
|
if ($row->{'Users'} < $USER_COUNT) {
|
||||||
|
print "$row->{'Domain'}," if ($LOGMODE ne "DEBUG");
|
||||||
|
print "$row->{'Application'} - $row->{'Domain'} - $row->{'Category'}, $row->{'CCI'}, $row->{'Users'}\n" if ($LOGMODE eq "DEBUG");
|
||||||
|
$EMAIL_CSV .= "$row->{'Domain'},";
|
||||||
|
push @blocklist, $row->{'Domain'};
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
print "\n\n" if $LOGMODE;
|
||||||
|
$EMAIL_CSV .= "\n";
|
||||||
|
}
|
||||||
|
return @blocklist;
|
||||||
|
}
|
||||||
|
|
||||||
|
#sub updateUrlList {
|
||||||
|
#my @domains = @{$_[0]};
|
||||||
|
#my $uri = "$NTSKP_TENANT/api/v1/updateUrlList?token=$NTSKP_TOKEN";
|
||||||
|
#my $request = HTTP::Tiny->new('default_headers' => \%HEADERS);
|
||||||
|
#$body = JSON::PP->new->encode({name => $NTSKP_URL_CATEGORY, list => \@domains});
|
||||||
|
#my $response = $request->post($uri, {'content' => $body});
|
||||||
|
#_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
#}
|
||||||
|
|
||||||
|
### Zscaler ###
|
||||||
|
|
||||||
|
sub zscaler {
|
||||||
|
my @domains = @{$_[0]};
|
||||||
|
|
||||||
|
### Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => \%HEADERS, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
### Push Domains
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
splice @domains, $ZS_MAX_DOMAINS if @domains > $ZS_MAX_DOMAINS;
|
||||||
|
$body = JSON::PP->new->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
|
||||||
|
### Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
_check_return($response->{'status'}, $response->{'content'}, $uri);
|
||||||
|
}
|
||||||
|
|
||||||
|
say "Running in $LOGMODE mode..." if $LOGMODE;
|
||||||
|
my @domains = netskope();
|
||||||
|
zscaler(\@domains);
|
||||||
|
mail_csv();
|
||||||
|
say "Completed." if $LOGMODE;
|
|
@ -0,0 +1,87 @@
|
||||||
|
#!/usr/bin/perl
|
||||||
|
use 5.024;
|
||||||
|
use strict;
|
||||||
|
use warnings;
|
||||||
|
use autodie;
|
||||||
|
use Config::Tiny;
|
||||||
|
use Time::HiRes qw(gettimeofday);
|
||||||
|
use POSIX qw(strftime);
|
||||||
|
use HTTP::Tiny;
|
||||||
|
use HTTP::CookieJar;
|
||||||
|
use JSON::PP;
|
||||||
|
use Text::CSV;
|
||||||
|
|
||||||
|
my $verbose = 1;
|
||||||
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
||||||
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
||||||
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
||||||
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
||||||
|
my $NTSKP_REPORTID = $config->{netskope}{NTSKP_REPORTID};
|
||||||
|
my $ZS_MAX_DOMAINS = $config->{zscaler}{ZS_MAX_DOMAINS};
|
||||||
|
my $ZS_BASE_URI = $config->{zscaler}{ZS_BASE_URI};
|
||||||
|
my $ZS_API_KEY = $config->{zscaler}{ZS_API_KEY};
|
||||||
|
my $ZS_API_USERNAME = $config->{zscaler}{ZS_API_USERNAME};
|
||||||
|
my $ZS_API_PASSWORD = $config->{zscaler}{ZS_API_PASSWORD};
|
||||||
|
my $ZS_CATEGORY_NAME = $config->{zscaler}{ZS_CATEGORY_NAME};
|
||||||
|
my $ZS_CATEGORY_DESC = $config->{zscaler}{ZS_CATEGORY_DESC};
|
||||||
|
|
||||||
|
say "Running...";
|
||||||
|
|
||||||
|
# Authenticate
|
||||||
|
my $now = int(gettimeofday * 1000);
|
||||||
|
my $n = substr($now, -6);
|
||||||
|
my $r = sprintf "%06d", $n >> 1;
|
||||||
|
my $key;
|
||||||
|
for my $i (0..length($n)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($n, $i, 1), 1);
|
||||||
|
}
|
||||||
|
for my $i (0..length($r)-1) {
|
||||||
|
$key .= substr($ZS_API_KEY, substr($r, $i, 1) + 2, 1);
|
||||||
|
}
|
||||||
|
my $uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
my $body = JSON::PP->new->space_after->encode({apiKey => $key, username => $ZS_API_USERNAME, password => $ZS_API_PASSWORD, timestamp => $now});
|
||||||
|
my $jar = HTTP::CookieJar->new;
|
||||||
|
my $request = HTTP::Tiny->new('default_headers' => {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, 'cookie_jar' => $jar);
|
||||||
|
my $response = $request->post($uri, {'content' => $body});
|
||||||
|
if ($verbose) {
|
||||||
|
say "POST $uri";
|
||||||
|
say "BODY $body";
|
||||||
|
say "HTTP " . $response->{'status'};
|
||||||
|
say "COOKIE " . $jar->cookie_header($ZS_BASE_URI);
|
||||||
|
}
|
||||||
|
|
||||||
|
# Get filter list id
|
||||||
|
$uri = "$ZS_BASE_URI/urlCategories/lite";
|
||||||
|
$response = $request->get($uri);
|
||||||
|
my $json = JSON::PP->new->utf8->decode($response->{'content'});
|
||||||
|
my $id;
|
||||||
|
for my $item (@{$json}) {
|
||||||
|
if (exists($item->{'configuredName'})) {
|
||||||
|
if ($item->{'configuredName'} eq $ZS_CATEGORY_NAME) {
|
||||||
|
$id = $item->{'id'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Push Domains
|
||||||
|
my @domains = ('secomtrust.net', 'baidupcs.com', 'cloud.baidu.com');
|
||||||
|
$uri = defined($id) ? "$ZS_BASE_URI/urlCategories/$id" : "$ZS_BASE_URI/urlCategories";
|
||||||
|
my $method = defined($id) ? "put" : "post";
|
||||||
|
my $description = "$ZS_CATEGORY_DESC\n\nLast Updated: " . strftime("%Y-%m-%d %H:%M:%S", localtime);
|
||||||
|
$#domains = $#domains >= $ZS_MAX_DOMAINS ? $ZS_MAX_DOMAINS : $#domains;
|
||||||
|
$body = JSON::PP->new->space_after->encode({configuredName => $ZS_CATEGORY_NAME, customCategory => 'true', superCategory => 'SECURITY', urls => \@domains, description => $description});
|
||||||
|
$response = $request->$method($uri, {'content' => $body});
|
||||||
|
if ($verbose) {
|
||||||
|
say uc($method) . " $uri";
|
||||||
|
say "BODY $body";
|
||||||
|
say "HTTP " . $response->{'status'};
|
||||||
|
say "RESPONSE " . $response->{'content'} if ($response->{'status'} =~ /^4/);
|
||||||
|
}
|
||||||
|
|
||||||
|
# Delete authenticadSession
|
||||||
|
$uri = "$ZS_BASE_URI/authenticatedSession";
|
||||||
|
$response = $request->delete($uri);
|
||||||
|
if ($verbose) {
|
||||||
|
say "DELETE $uri";
|
||||||
|
say "HTTP " . $response->{'status'};
|
||||||
|
}
|
Loading…
Reference in New Issue