70 lines
1.8 KiB
Perl
Executable File
70 lines
1.8 KiB
Perl
Executable File
#!/usr/bin/perl -w
|
|
use strict;
|
|
use warnings;
|
|
use autodie;
|
|
use POSIX qw(strftime);
|
|
use Config::Tiny;
|
|
use HTTP::Tiny;
|
|
use Cpanel::JSON::XS;
|
|
|
|
my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
|
|
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
|
|
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
|
|
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
|
|
my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
|
|
my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
|
|
my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};
|
|
|
|
my $uri;
|
|
my $skip = 0;
|
|
my $response;
|
|
my $json;
|
|
my $data;
|
|
my $length;
|
|
my $domain;
|
|
my $cci;
|
|
|
|
my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt";
|
|
print "File: $file_out\n";
|
|
print "Tenant: $NTSKP_TENANT\n";
|
|
|
|
while ($skip < 500000) {
|
|
$uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip";
|
|
$response = HTTP::Tiny->new->get($uri);
|
|
print "HTTP: $response->{status} $response->{reason}\n";
|
|
$json = Cpanel::JSON::XS->new->utf8->decode($response->{content});
|
|
print "API: $json->{'status'}\n";
|
|
$data = $json->{'data'};
|
|
|
|
$length = (@{$data});
|
|
if ($length == 0) {
|
|
print "All data collected\n";
|
|
last;
|
|
}
|
|
|
|
open my $fh_out, ">>", $file_out;
|
|
for (my $i = 0; $i < $length; $i++) {
|
|
if (!$data->[$i]->{'domain'}) {
|
|
my $url = $data->[$i]->{'url'};
|
|
$url =~ s!^https?://(?:www\.)?!!i;
|
|
$url =~ s!/.*!!;
|
|
$url =~ s/[\?\#\:].*//;
|
|
$domain = $url;
|
|
} else {
|
|
$domain = $data->[$i]->{'domain'};
|
|
}
|
|
if ($data->[$i]->{'cci'}) {
|
|
$cci = $data->[$i]->{'cci'};
|
|
} else {
|
|
$cci = 'none';
|
|
}
|
|
|
|
my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
|
|
print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n";
|
|
}
|
|
close $fh_out;
|
|
$skip += 5000;
|
|
#print "Next batch $skip\n";
|
|
}
|
|
print "Done\n";
|