#!/usr/bin/perl -w
use strict;
use warnings;
use autodie;
use POSIX qw(strftime);
use Config::Tiny;
use HTTP::Tiny;
use Cpanel::JSON::XS;

my $CONFIG_FILE = "/home/mischa/netskope/netskope.cnf";
my $config = Config::Tiny->read($CONFIG_FILE, 'utf8');
my $NTSKP_TENANT = $config->{netskope}{NTSKP_TENANT};
my $NTSKP_TOKEN = $config->{netskope}{NTSKP_TOKEN};
my $NTSKP_PERIOD = $config->{netskope}{NTSKP_PERIOD};
my $NTSKP_SCORE = $config->{netskope}{NTSKP_SCORE};
my $NTSKP_CATEGORIES = $config->{netskope}{NTSKP_CATEGORIES};

my $uri;
my $skip = 0;
my $response;
my $json;
my $data;
my $length;
my $domain;
my $cci;

my $file_out = "extracted-" . strftime("%Y%m%d", localtime) . ".txt";
print "File: $file_out\n";
print "Tenant: $NTSKP_TENANT\n";

while ($skip < 500000) {
	$uri = "$NTSKP_TENANT/api/v1/events?token=$NTSKP_TOKEN&type=page&timeperiod=$NTSKP_PERIOD&skip=$skip";
	$response = HTTP::Tiny->new->get($uri);
	print "HTTP: $response->{status} $response->{reason}\n";
	$json = Cpanel::JSON::XS->new->utf8->decode($response->{content});
	print "API: $json->{'status'}\n";
	$data = $json->{'data'};

	$length = (@{$data});
	if ($length == 0) {
		print "All data collected\n";
		last;
	}

	open my $fh_out, ">>", $file_out;
	for (my $i = 0; $i < $length; $i++) {
		if (!$data->[$i]->{'domain'}) {
			my $url = $data->[$i]->{'url'};
			$url =~ s!^https?://(?:www\.)?!!i;
			$url =~ s!/.*!!;
			$url =~ s/[\?\#\:].*//;
			$domain	= $url;
		} else {
			$domain = $data->[$i]->{'domain'};
		}
		if ($data->[$i]->{'cci'}) {
			$cci = $data->[$i]->{'cci'};
		} else {
			$cci = 'none';
		}
	
		my $timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime($data->[$i]->{'timestamp'}));
		print $fh_out "$timestamp,$domain,$cci,$data->[$i]->{'category'},$data->[$i]->{'ccl'},$data->[$i]->{'user'}\n";
	}
	close $fh_out;
	$skip += 5000;
	#print "Next batch $skip\n";
}
print "Done\n";