Merge pull request #37 from usit-gd/feature-frontend

Feature: New web frontend

.gitignore

@@ -1,3 +1,4 @@
 *.exe
-server/web/web
-server/jobrunner/jobrunner
+server/service/service
+server/website/libs/*
+!server/website/libs/download_libraries.sh

client/nivlheim_client

@@ -186,8 +186,8 @@ if ($opt{debug}) {
 	print "Using CA file $real_ca_file\n";
 }
 
+# Verify access to the certificate files and directory
 my $have_cert = (-f $opt{cert_file} && -f $opt{key_file});
-
 if ($have_cert) {
 	if (-f $opt{cert_file} && ! -w $opt{cert_file}) {
 		printlog "Must have write access to $opt{cert_file}.";
@@ -197,6 +197,13 @@ if ($have_cert) {
 		printlog "Must have write access to $opt{key_file}.";
 		exit 1;
 	}
+} else {
+	# Verify that the directory exists
+	my $dirname = dirname($opt{cert_file});
+	if (!-d $dirname) {
+		mkpath($dirname, {error => \my $err});
+		if (!-d $dirname) { printlog "The path $dirname doesn't exist, and I am unable to create it.\n"; exit 1; }
+	}
 }
 
 # Is there a client certificate present? Test if it works
@@ -256,17 +263,20 @@ if (!$have_cert) {
 elsif ($have_cert && !$cert_works) {
 	eval {
 		printlog "Trying to renew the certificate.\n";
-		# Verify that the directory exists
-		my $dirname = dirname($opt{cert_file});
-		if (!-d $dirname) {
-			mkpath($dirname, {error => \my $err});
-			if (!-d $dirname) { printlog "The path $dirname doesn't exist, and I am unable to create it.\n"; exit 1; }
-		}
 		my $cert = undef;
 		my $key = undef;
 		# Request a new certificate on the grounds that we already have an old one
 		my $response = http_get($server_url . 'secure/renewcert');
 		($cert, $key) = parse_certificate_response($response);
+		if (defined($cert) && defined($key)) {
+			printlog "Successfully renewed the certificate.\n";
+		} else {
+			printlog "Wasn't able to renew the certificate. Requesting a new one instead...\n";
+			my $hostname = `hostname -f`;
+			chop $hostname;
+			my $response = http_get($server_url . "reqcert?hostname=$hostname");
+			($cert, $key) = parse_certificate_response($response);
+		}
 		# Did it work?
 		if (defined($cert) && defined($key)) {
 			printlog "Successfully renewed the certificate.\n";
@@ -280,7 +290,7 @@ elsif ($have_cert && !$cert_works) {
 			printlog "Received and stored a new certificate.\n";
 		}
 		else {
-			printlog "Wasn't able to renew the certificate.\n";
+			printlog "Did not receive any certificate.\n";
 			exit 1;
 		}
 	};

rpm/buildrpm.sh

@@ -20,7 +20,7 @@ then
 	exit 1
 fi
 cp *.spec $BUILDDIR/SPECS/
-[ -f *.patch ] && cp *.patch $BUILDDIR/SOURCES/
+cp *.patch $BUILDDIR/SOURCES/ 2>/dev/null
 SPEC=`eval echo $BUILDDIR/SPECS/*.spec`
 echo "buildrpm: Spec file = $SPEC"
 

rpm/nivlheim.spec

@@ -13,11 +13,11 @@ License:  GPLv3+
 URL:      https://github.com/usit-gd/nivlheim
 Source0:  https://github.com/usit-gd/nivlheim/archive/%{getenv:GIT_BRANCH}.tar.gz
 
+BuildRequires: npm(handlebars)
 BuildRequires: perl(Archive::Tar)
 BuildRequires: perl(Archive::Zip)
 BuildRequires: perl(CGI)
 BuildRequires: perl(Crypt::OpenSSL::X509)
-BuildRequires: perl(DateTime)
 BuildRequires: perl(DBD::Pg)
 BuildRequires: perl(DBI)
 BuildRequires: perl(Encode)
@@ -77,7 +77,6 @@ Requires: perl(Archive::Tar)
 Requires: perl(Archive::Zip)
 Requires: perl(CGI)
 Requires: perl(Crypt::OpenSSL::X509)
-Requires: perl(DateTime)
 Requires: perl(DBD::Pg)
 Requires: perl(DBI)
 Requires: perl(Encode)
@@ -112,9 +111,9 @@ mkdir -p %{buildroot}%{_sbindir}
 mkdir -p %{buildroot}%{_sysconfdir}/nivlheim
 mkdir -p %{buildroot}%{_sysconfdir}/httpd/conf.d
 mkdir -p %{buildroot}%{_localstatedir}/nivlheim/go/{src,pkg,bin}
-mkdir -p %{buildroot}/var/www/nivlheim/templates
+mkdir -p %{buildroot}/var/www/nivlheim
 mkdir -p %{buildroot}/var/www/cgi-bin/secure
-mkdir -p %{buildroot}/var/www/html/static
+mkdir -p %{buildroot}/var/www/html
 mkdir -p %{buildroot}/var/log/nivlheim
 mkdir -p %{buildroot}%{_unitdir}
 mkdir -p %{buildroot}%{_sysconfdir}/logrotate.d
@@ -135,12 +134,16 @@ install -p -m 0755 server/cgi/parsefile %{buildroot}/var/www/cgi-bin/
 install -p -m 0644 server/nivlheim.service %{buildroot}%{_unitdir}/%{name}.service
 install -p -m 0644 server/logrotate.conf %{buildroot}%{_sysconfdir}/logrotate.d/%{name}-server
 install -p -m 0755 -D client/cron_hourly %{buildroot}%{_sysconfdir}/cron.hourly/nivlheim_client
-cp -r server/web %{buildroot}%{_localstatedir}/nivlheim/go/src/
-cp -r server/jobrunner %{buildroot}%{_localstatedir}/nivlheim/go/src/
-cp server/templates/* %{buildroot}/var/www/nivlheim/templates/
-cp -r server/static/* %{buildroot}%{_localstatedir}/www/html/static/
+cp -r server/service %{buildroot}%{_localstatedir}/nivlheim/go/src/
 echo %{version} > %{buildroot}%{_sysconfdir}/nivlheim/version
 
+# Compile web templates
+handlebars server/website/templates --min -f server/website/js/templates.js
+
+# Copy static website files, excluding files that are only for development
+rm -rf server/website/mockapi server/website/templates
+cp -r server/website/* %{buildroot}%{_localstatedir}/www/html/
+
 %check
 perl -c %{buildroot}%{_sbindir}/nivlheim_client
 perl -c %{buildroot}/var/www/cgi-bin/secure/renewcert
@@ -180,7 +183,7 @@ rm -rf %{buildroot}
 %dir /var/log/nivlheim
 /var/www/nivlheim
 /var/www/cgi-bin
-/var/www/html/static
+/var/www/html/*
 %attr(0644, root, apache) /var/www/nivlheim/log4perl.conf
 %attr(0755, root, root) %{_localstatedir}/nivlheim/setup.sh
 %{_localstatedir}/nivlheim
@@ -196,6 +199,9 @@ rm -rf %{buildroot}
 %systemd_postun_with_restart %{name}.service
 
 %changelog
+* Fri Feb 23 2018 Øyvind Hagberg <[email protected]> - 0.1.4-20180223
+- New web frontend, installs in /var/www/html. frontpage.cgi is gone.
+
 * Fri Jan 05 2018 Øyvind Hagberg <[email protected]> - 0.1.1-20180105
 - Removed dependencies on the missing parent package "nivlheim",
   since it isn't being build anymore.

rpm/test_packages.sh

@@ -5,6 +5,7 @@ set -x
 # It should signal success by outputting "END_TO_END_SUCCESS" if and only if
 # the test(s) succeeded.
 
+# Install the packages. Different methods on Fedora and CentOS.
 if [ -f /etc/fedora-release ]; then
 	sudo dnf copr -y enable oyvindh/Nivlheim-test
 	sudo dnf install -y nivlheim-client nivlheim-server || touch installerror
@@ -14,20 +15,41 @@ elif [ -f /etc/centos-release ]; then
 		https://copr.fedorainfracloud.org/coprs/oyvindh/Nivlheim-test/repo/epel-7/oyvindh-Nivlheim-test-epel-7.repo
 	sudo yum install -y nivlheim-client nivlheim-server || touch installerror
 fi
-
 if [ -f installerror ]; then
 	echo "Package installation failed."
 	exit
 fi
 
-if [ $(curl -s -k https://localhost/ | grep -c "Nivlheim") -eq 0 ]; then
+# Check that the home page is being served
+if [ $(curl -sSk https://localhost/ | tee /tmp/homepage | grep -c "<title>Nivlheim</title>") -eq 0 ]; then
 	echo "The web server isn't properly configured and running."
 	exit
 fi
+# 3rd party libraries
+for URL in $(perl -ne 'm!"(libs/.*?)"!&&print "$1\n"' < /tmp/homepage);
+do
+	if ! curl -sSkfo /dev/null "https://localhost/$URL"; then
+		echo "The web server returns an error code for $URL"
+		exit
+	fi
+done
 
+# Check that the API is available through the main web server
+if ! curl -sSkfo /dev/null https://localhost/api/v0/status; then
+	echo "The API is unavailable."
+	exit
+fi
+
+# Configure the client to use the server at localhost
 echo "server=localhost" | sudo tee -a /etc/nivlheim/client.conf
+# Run the client, it will be put on waiting list for a certificate
 sudo /usr/sbin/nivlheim_client
-sudo -u apache psql -c 'update waiting_for_approval set approved=true;'
+# Approve the client, using the API
+ID=`curl -sS 'http://localhost:4040/api/v0/awaitingApproval?fields=approvalId'|perl -ne 'print $1 if /"approvalId":\s+(\d+)/'`
+curl -X PUT -sS "http://localhost:4040/api/v0/awaitingApproval/$ID?hostname=abcdef"
+
+# Run the client again, this time it will receive a certificate
+# and post data into the system
 sudo /usr/sbin/nivlheim_client
 if [ ! -f /var/nivlheim/my.crt ]; then
 	echo "Certificate generation failed."
@@ -38,7 +60,8 @@ fi
 OK=0
 for try in {1..20}; do
 	sleep 3
-	if [ $(curl -s -k https://localhost/ | grep -c "novalocal") -gt 0 ]; then
+	# Query the API for the new machine
+	if [ $(curl -sS 'http://localhost:4040/api/v0/hostlist?fields=hostname' | grep -c "abcdef") -gt 0 ]; then
 		OK=1
 		break
 	fi

server/cgi/parsefile

@@ -9,7 +9,6 @@ use Log::Log4perl;
 use Net::DNS;
 
 sub parse_query_string();
-sub uts_to_iso8601($);
 
 # Restrict access to only localhost
 my $ip = $ENV{'REMOTE_ADDR'};
@@ -52,8 +51,8 @@ eval{
 	$dbh->{RaiseError} = 1;
 
 	my $sth = $dbh->prepare("SELECT filename, content, " .
-		"received, is_command, certcn, ipaddr, certfp, clientversion " .
-		"FROM files WHERE fileid=?");
+		"received, is_command, certcn, ipaddr, certfp, clientversion, " .
+		"clienthostname FROM files WHERE fileid=?");
 	$sth->bind_param(1, $fileid);
 	my $rv = $sth->execute;
 	my $row = $sth->fetchrow_hashref;
@@ -287,10 +286,13 @@ if ($@) {
 		$logger->error($dbh->{Statement});
 	}
 	$dbh->rollback;
+	$dbh->disconnect;
 	print CGI::header(
 		-type => 'text/plain',
 		-status => '500 Internal Server Error'
 	);
+	print $err;
+	exit 1;
 };
 
 # Clean up
@@ -308,9 +310,3 @@ sub parse_query_string() {
 	}
 	return %result;
 }
-
-sub uts_to_iso8601($) {
-	my $uts = shift;
-	my $date = DateTime->from_epoch(epoch => $uts, time_zone => 'UTC');
-	return $date->ymd().q{T}.$date->hms().'Z';
-}

server/cgi/processarchive

@@ -5,18 +5,18 @@ use warnings;
 use Archive::Zip qw( :ERROR_CODES :CONSTANTS );
 use CGI;
 use DBI;
-use DateTime;
 use Encode qw(from_to encode_utf8 decode_utf8);
 use File::Basename;
 use File::Find;
 use File::Temp qw/ tempdir /;
 use IPC::Open3;
 use Log::Log4perl;
+use POSIX qw(strftime);
 use Symbol qw(gensym);
 
 sub parse_query_string();
 sub run_command($);
-sub uts_to_iso8601($);
+sub uts_to_rfc3339($);
 
 # Restrict access to only localhost
 my $ip = $ENV{'REMOTE_ADDR'};
@@ -118,7 +118,7 @@ eval{
 	}
 	close($F);
 
-	my $iso_received = uts_to_iso8601($meta{'received'});
+	my $iso_received = uts_to_rfc3339($meta{'received'});
 
 	my $sth = $dbh->prepare("INSERT INTO files(ipaddr,clienthostname,"
 		."certcn,certfp,filename,received,mtime,content,is_command,clientversion) "
@@ -135,7 +135,7 @@ eval{
 
 		# Make an ISO timestamp from the modified time of the file
 		my $mtime = (stat($File::Find::name))[9];
-		my $iso_mtime = uts_to_iso8601($mtime);
+		my $iso_mtime = uts_to_rfc3339($mtime);
 
 		# Read the content
 		my ($content, $originalfilename);
@@ -192,10 +192,13 @@ eval{
 if ($@) {
 	$logger->error($@);
 	$dbh->rollback;
+	$dbh->disconnect;
 	print CGI::header(
 		-type => 'text/plain',
 		-status => '500 Internal Server Error'
 	);
+	print $@;
+	exit 1;
 };
 
 # Clean up
@@ -214,10 +217,9 @@ sub parse_query_string() {
 	return %result;
 }
 
-sub uts_to_iso8601($) {
+sub uts_to_rfc3339($) {
 	my $uts = shift;
-	my $date = DateTime->from_epoch(epoch => $uts, time_zone => 'UTC');
-	return $date->ymd().q{T}.$date->hms().'Z';
+	return strftime("%Y-%m-%dT%H:%M:%SZ",gmtime($uts));
 }
 
 sub run_command($) {

server/cgi/renewcert

@@ -42,7 +42,7 @@ print "Content-Type: text/plain\n\n";
 
 # There is a race condition with openssl, so until that is solved
 # there can be only one instance of this script running at any time.
-if (Proc::PID::File->running("dir" => "/tmp")) {
+if (Proc::PID::File->running({"dir" => "/tmp", "name" => "reqcert"})) {
 	print "Too busy, please try again later.\n";
 	exit;
 }

server/cgi/reqcert

@@ -103,7 +103,7 @@ print "Content-Type: text/plain\n\n";
 
 # There is a race condition with openssl, so until that is solved
 # there can be only one instance of this script running at any time.
-if (Proc::PID::File->running("dir" => "/tmp")) {
+if (Proc::PID::File->running({"dir" => "/tmp", "name" => "reqcert"})) {
 	print "Too busy, please try again later.\n";
 	$logger->debug("Too busy, let's hope they try again later.");
 	exit;

server/httpd_ssl.conf

@@ -28,11 +28,9 @@ SSLVerifyDepth  10
     SSLRequire %{SSL_CLIENT_VERIFY} eq "SUCCESS"
 </Directory>
 
-<Location ~ "/">
-    DirectoryIndex cgi-bin/frontpage.cgi
+<Location "/api/">
+    ProxyPass "http://127.0.0.1:4040/api/"
 </Location>
-ScriptAlias "/search" "/var/www/cgi-bin/frontpage.cgi"
-ScriptAlias "/browse" "/var/www/cgi-bin/frontpage.cgi"
 
 </VirtualHost>