Client compatibility, PKCS8, settings page, cron

Modified the client to be compatible with the legacy version of the
server software.
The client generates an additional file with the certificate key
in PKCS8 format. This helps #32.
Modified the client cron job to run every 5 minutes, and the client
will exit if less than an hour since last successful run. Fixes #20.
Moved approve/deny web gui to the settings page.
Improved the webdesign/look on the settings page.

client/cronjob

@@ -0,0 +1 @@
+*/5 * * * *   root  /usr/sbin/nivlheim_client -minperiod 3500 -sleeprandom 300 > /dev/null

client/nivlheim_client

@@ -25,8 +25,9 @@ use IO::Socket::INET6;
 use Net::DNS;
 use Archive::Tar;
 use HTTP::Request::Common;
+use Sys::Hostname;
 use Sys::Syslog qw(:standard :macros);
-use File::Path;
+use File::Path qw(remove_tree);
 use File::Basename;
 use Getopt::Long qw(:config no_ignore_case);
 
@@ -38,6 +39,8 @@ sub printlog($);
 sub shortencmd($);
 sub reverse_dns_lookup($$);
 sub parse_certificate_response($);
+sub createPKCS8();
+sub getpassword();
 
 # Options with default values
 my %defaultopt = (
@@ -50,16 +53,22 @@ my %defaultopt = (
 	'debug'     => 0,  # debugging / verbose output
 	'help'      => 0,  # display help output
 	'version'   => 0,  # plugin version info
+	'sleeprandom' => 0,
+	'minperiod' => 0,
 	);
 
 # Version information
 my $NAME    = 'nivlheim_client';
 my $AUTHOR  = 'Øyvind Hagberg';
 my $CONTACT = '[email protected]';
 my $RIGHTS  = 'USIT/IT-DRIFT/GD/GID, University of Oslo, Norway';
-open(my $F, "/etc/nivlheim/version");
-chomp(my $VERSION = <$F>);
-close($F);
+my $VERSION;
+if (open(my $F, "/etc/nivlheim/version")) {
+	chomp($VERSION = <$F>);
+	close($F);
+} else {
+	$VERSION = "unknown_version";
+}
 
 # Usage text
 my $USAGE = <<"END_USAGE";
@@ -96,6 +105,8 @@ GetOptions('c|config=s'  => \$opt{config},
 	   'ssl-ca=s'    => \$opt{ca_file},
 	   'ssl-cert=s'  => \$opt{cert_file},
 	   'ssl-key=s'   => \$opt{key_file},
+	   'sleeprandom' => \$opt{sleeprandom},
+	   'minperiod'   => \$opt{minperiod},
 	   'd|debug'     => \$opt{debug},
 	   'h|help'      => \$opt{help},
 	   'V|version'   => \$opt{version},
@@ -113,12 +124,38 @@ if ($opt{version}) {
     exit 0;
 }
 
+# sleep if cmdline parameter says so
+if ($opt{sleeprandom}) {
+	# Take the sum of the characters in the hostname,
+	# divide by the parameter value and take the remainder.
+	# Sleep that many seconds.
+	# (i.e. a "random" delay but the same every time)
+	sleep unpack("%32W*",hostname) % $opt{sleeprandom};
+}
+
+# minimum period between successful runs
+if ($opt{minperiod}) {
+	my @stat = stat "/var/run/nivlheim_client_last_run";
+	exit 0 if ((time()-$stat[9]) < $opt{minperiod});
+}
+
 # Log to stdout or syslog depending on whether we have a TTY
 if (!-t STDOUT) {
 	# There's no TTY. Open syslog
 	openlog('nivlheim', '', Sys::Syslog::LOG_DAEMON);
 }
 
+# tempdir
+my $tempdir = "/tmp/nivlheim$$";
+mkdir($tempdir);
+END {
+	print "Cleanup\n" if ($opt{debug});
+	remove_tree($tempdir);
+	if (!-t STDOUT) {
+		closelog();
+	}
+}
+
 # Read the config file
 $opt{config} ||= $defaultopt{config};
 my $configfile;
@@ -232,9 +269,7 @@ if (!$have_cert || !$cert_works) {
 if (!$have_cert) {
 	eval {
 		printlog "Requesting a certificate..\n";
-		my $hostname = `hostname -f`;
-		chop $hostname;
-		my $response = http_get($server_url . "reqcert?hostname=$hostname");
+		my $response = http_get($server_url . "reqcert?hostname=".hostname);
 		my ($cert, $key) = parse_certificate_response($response);
 		if (defined($cert) && defined($key)) {
 			printlog "Received a certificate.\n";
@@ -245,6 +280,7 @@ if (!$have_cert) {
 			print $F "$key\n";
 			close($F);
 			chmod(0600, $opt{key_file});
+			createPKCS8();
 			printlog "Received and stored a new certificate.\n";
 		}
 		else {
@@ -269,9 +305,7 @@ elsif ($have_cert && !$cert_works) {
 			printlog "Successfully renewed the certificate.\n";
 		} else {
 			printlog "Wasn't able to renew the certificate. Requesting a new one instead...\n";
-			my $hostname = `hostname -f`;
-			chop $hostname;
-			my $response = http_get($server_url . "reqcert?hostname=$hostname");
+			my $response = http_get($server_url . "reqcert?hostname=".hostname);
 			($cert, $key) = parse_certificate_response($response);
 		}
 		# Did it work?
@@ -284,6 +318,7 @@ elsif ($have_cert && !$cert_works) {
 			print $F "$key\n";
 			close($F);
 			chmod(0600, $opt{key_file});
+			createPKCS8();
 			printlog "Received and stored a new certificate.\n";
 		}
 		else {
@@ -391,14 +426,21 @@ foreach my $alias (@aliaslist) {
 }
 
 # Compress and save the tar file
-my $tarfile = "/tmp/filelog.tgz";
+my $tarfile = "$tempdir/archive.tgz";
 unlink($tarfile); # In case it already exists
 $tar->write($tarfile, Archive::Tar::COMPRESS_GZIP());
+print "Wrote archive file\n" if ($opt{debug});
 
 # Create a signature for the tar file
 my $signaturefile = "$tarfile.sign";
 unlink($signaturefile); # In case it already exists
-system("openssl dgst -sha256 -sign $opt{key_file} -out $signaturefile $tarfile");
+#system("openssl dgst -sha256 -sign $opt{key_file} -out $signaturefile $tarfile");
+# support for certificate files that have a password,
+# generated by older versions of the server software
+open(my $F, "| openssl dgst -sha256 -sign $opt{key_file} -passin stdin -out $signaturefile $tarfile");
+print $F getpassword();
+close($F);
+print "Signed the archive file\n" if ($opt{debug});
 
 # nonce
 my $nonce = 0;
@@ -411,10 +453,8 @@ if (-r $noncefile) {
 }
 
 # POST all of it
-my $myhostname = `hostname -f`;
-$myhostname =~ s/[\r\n]//g;
 my %postdata = (
-	'hostname' => $myhostname,
+	'hostname' => hostname,
 	'archive' => [$tarfile],
 	'signature' => [$signaturefile],
 	'version' => $VERSION,
@@ -440,13 +480,6 @@ eval {
 };
 printlog $@ if ($@);
 
-# Clean up
-unlink($tarfile);
-unlink($signaturefile);
-if (!-t STDOUT) {
-	closelog();
-}
-
 #---------------- end --------------
 
 # Establish an SSL connection to the server.
@@ -528,6 +561,10 @@ sub ssl_connect($$$) {
 	}
 }
 
+# The password for the private rsa key. Only here for compatibility with
+# a legacy version of the software, later the keys will be passwordless.
+sub getpassword() { return "passord123"; }
+
 sub split_url($) {
 	my $url = shift;
 	# FQDN
@@ -658,7 +695,7 @@ sub http_post($$) {
 	$req =~ s/^POST (.*)$/POST $1 HTTP\/1.0/m;
 
 	# Convert the line endings to two-byte CR LF
-	my ($headers, $rest) = split /\n\n/, $req;
+	my ($headers, $rest) = split /\n\n/, $req, 2;
 	$headers =~ s/\n/\r\n/gs;
 	$req = $headers . "\r\n\r\n" . $rest;
 
@@ -782,3 +819,12 @@ sub parse_certificate_response($) {
 	}
 	return ($cert, $key);
 }
+
+sub createPKCS8() {
+	#system("openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in /var/nivlheim/my.key -out /var/nivlheim/pkcs8.key");
+	# support for certificate files that have a password,
+	# generated by older versions of the server software
+	open(my $F, "| openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -passin stdin -in /var/nivlheim/my.key -out /var/nivlheim/pkcs8.key");
+	print $F getpassword();
+	close($F);
+}

rpm/nivlheim.spec

@@ -44,6 +44,7 @@ BuildRequires: perl(Net::DNS)
 BuildRequires: perl(Net::IP)
 BuildRequires: perl(Proc::PID::File)
 BuildRequires: perl(Socket)
+BuildRequires: perl(Sys::Hostname)
 BuildRequires: perl(Sys::Syslog)
 BuildRequires: perl(Time::Piece)
 BuildRequires: systemd, golang, git
@@ -66,6 +67,7 @@ Requires: perl(IO::Socket::INET6)
 Requires: perl(IO::Socket::SSL)
 Requires: perl(Net::DNS)
 Requires: perl(Socket)
+Requires: perl(Sys::Hostname)
 Requires: perl(Sys::Syslog)
 
 %package server
@@ -149,7 +151,7 @@ install -p -m 0644 server/init.sql %{buildroot}%{_localstatedir}/nivlheim/
 install -p -m 0755 server/cgi/processarchive %{buildroot}/var/www/cgi-bin/
 install -p -m 0644 server/nivlheim.service %{buildroot}%{_unitdir}/%{name}.service
 install -p -m 0644 server/logrotate.conf %{buildroot}%{_sysconfdir}/logrotate.d/%{name}-server
-install -p -m 0755 -D client/cron_hourly %{buildroot}%{_sysconfdir}/cron.hourly/nivlheim_client
+install -p -m 0644 -D client/cronjob %{buildroot}%{_sysconfdir}/cron.d/nivlheim_client
 rm -rf server/website/mockapi server/website/templates
 cp -a server/website/* %{buildroot}%{_localstatedir}/www/html/
 install -p -m 0755 gopath/bin/service %{buildroot}%{_sbindir}/nivlheim_service
@@ -177,7 +179,7 @@ rm -rf %{buildroot}
 %{_sbindir}/nivlheim_client
 %config %{_sysconfdir}/nivlheim/version
 %config(noreplace) %{_sysconfdir}/nivlheim/client.conf
-%{_sysconfdir}/cron.hourly/nivlheim_client
+%{_sysconfdir}/cron.d/nivlheim_client
 
 %files server
 %defattr(-, root, root, -)
@@ -209,6 +211,9 @@ rm -rf %{buildroot}
 %systemd_postun_with_restart %{name}.service
 
 %changelog
+* Wed Apr 18 2018 Øyvind Hagberg <[email protected]> - 0.6.0-20180418
+- The client requires perl(Sys::Hostname), and has a new cron job
+
 * Tue Mar 27 2018 Øyvind Hagberg <[email protected]> - 0.4.0-20180327
 - Removed the cgi script "parsefile"
 

rpm/run_tests_on_vms.sh

@@ -14,13 +14,16 @@
 source ~/keystone_rc.sh # provides environment variables with authentication information
 source ~/github_rc.sh # provides GitHub API token
 
+echo "Region: $OS_REGION_NAME"
+
 KEYPAIRNAME="jenkins_key"
 SECGROUP=""
-while getopts "s:k:" option; do
+while getopts "s:k:f" option; do
 	case "${option}"
 	in
 		k) KEYPAIRNAME=${OPTARG};;
 		s) SECGROUP="--security-group ${OPTARG}";;
+		f) FAILFAST=1
 	esac
 done
 
@@ -67,7 +70,7 @@ for IMAGE in "${IMAGES[@]}"; do
 	NAME="voyager"
 	openstack server delete --wait $NAME 2>/dev/null # just to be sure
 
-	# UH-IaaS suddenly decided to keep more than 1 active image with the same name.
+	# UH-IaaS may keep more than 1 active image with the same name.
 	# That means we can't use the image name for the "create server"-command.
 	# This is a workaround.
 	ID=$(openstack image list | grep "$IMAGE" | grep active | head -1 | cut -d '|' -f 2 | xargs)
@@ -99,15 +102,6 @@ for IMAGE in "${IMAGES[@]}"; do
 				OK=1
 				break
 			fi
-			# UH-IaaS is unreliable, sometimes you have to stop/start the VM
-			# before it will let you connect
-			if [[ $(expr $try % 20) -eq 0 ]]; then
-				echo -n "o"
-				openstack server stop $NAME
-				sleep 10
-				echo -n "O"
-				openstack server start $NAME
-			fi
 			sleep 5
 			echo -n "."
 		done
@@ -143,6 +137,10 @@ for IMAGE in "${IMAGES[@]}"; do
 
 		if grep -s "FAIL" "$LOGFILE"; then
 			echo $(grep -c "FAIL" "$LOGFILE") "FAIL(s)"
+			if [[ $FAILFAST -eq 1 ]]; then
+				openstack server delete $NAME
+				exit 1
+			fi
 		fi
 
 		if [[ $(whoami) == "jenkins" ]]; then

rpm/test_packages.sh

@@ -69,6 +69,8 @@ for try in {1..20}; do
 done
 if [ $OK -eq 0 ]; then
 	echo "Home page does not show the new machine."
+	grep -A1 "ERROR" /var/log/nivlheim/system.log
+	sudo grep "cgi:error" /var/log/httpd/error_log | grep -v 'random state'
 	exit 1
 fi
 

server/service/api_ipranges_test.go

@@ -4,11 +4,16 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 )
 
 func TestApiMethodIpRanges(t *testing.T) {
+	if os.Getenv("NOPOSTGRES") != "" {
+		t.Log("No Postgres, skipping test")
+		return
+	}
 	type apiCall struct {
 		methodAndPath, body string
 		expectStatus        int

server/website/js/functions.js

@@ -225,7 +225,6 @@ function restDelete(element, apiPath) {
 	});
 }
 
-//----====----====----====-- Frontpage --====----====----====
 function approve(id) {
 	$.ajax({
 		url : getAPIURLprefix()+'/api/v0/awaitingApproval/'
@@ -251,7 +250,7 @@ function deny(id) {
 	});
 }
 
-
+//----====----====----====-- Frontpage --====----====----====
 let reloadingTimeout = 0;
 
 function autoReloadStatus() {

server/website/js/index.js

@@ -123,11 +123,6 @@ function showFrontPage() {
 			if(e.keyCode===13){newSearch();}
 		});
 		autoReloadStatus();
-		APIcall(
-			//"mockapi/awaiting_approval.json",
-			"/api/v0/awaitingApproval"+
-			"?fields=hostname,reversedns,ipaddress,approvalId",
-			"awaiting_approval", $('#placeholder_approval'));
 	});
 }
 
@@ -325,5 +320,10 @@ function settingsPage() {
 		.done(function(){
 			attachHandlersToForms();
 		});
+		APIcall(
+			//"mockapi/awaiting_approval.json",
+			"/api/v0/awaitingApproval"+
+			"?fields=hostname,reversedns,ipaddress,approvalId",
+			"awaiting_approval", $('#placeholder_approval'));
 	});
 }