Skip to content

Commit

Permalink
Merge pull request #61 from usit-gd/minor-fixes-dec2018
Browse files Browse the repository at this point in the history
Various bugfixes and improvements
  • Loading branch information
oyvindhagberg authored Dec 7, 2018
2 parents d9a03a4 + 0fa3fa1 commit a27b432
Show file tree
Hide file tree
Showing 14 changed files with 254 additions and 108 deletions.
8 changes: 1 addition & 7 deletions client/nivlheim_client
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,7 @@ my $NAME = 'nivlheim_client';
my $AUTHOR = 'Øyvind Hagberg';
my $CONTACT = '[email protected]';
my $RIGHTS = 'USIT/IT-DRIFT/GD/GID, University of Oslo, Norway';
my $VERSION;
if (open(my $F, "/etc/nivlheim/version")) {
chomp($VERSION = <$F>);
close($F);
} else {
$VERSION = "unknown_version";
}
my $VERSION = '0.10.1';

# Usage text
my $USAGE = <<"END_USAGE";
Expand Down
11 changes: 8 additions & 3 deletions rpm/nivlheim.spec
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ This package contains the server components of Nivlheim.
%setup -q -T -b 3 -n net-master
%autosetup -D -n %{name}-%{getenv:GIT_BRANCH}

# disable building of the debug package.
# avoids the error debuginfo-without-sources from rpmlint
%define debug_package %{nil}

%build
# Compile web templates
handlebars server/website/templates --min -f server/website/js/templates.js
Expand All @@ -134,8 +138,10 @@ mv ../oauth2-master $GOPATH/src/golang.org/x/oauth2
pushd $GOPATH/src/github.com/usit-gd/nivlheim/server/service
NONETWORK=1 NOPOSTGRES=1 go test -v
rm -f $GOPATH/bin/*
# Fix for the error "No build ID note found in ..."
go install -ldflags=-linkmode=external
# The linkmode=external flag is a fix for the error "No build ID note found in ...".
# The -w flag disables debug info generation, and -s omits the symbol table.
# By using -w and -s we avoid the unstripped-binary-or-object warning from rpmlint
go install -ldflags='-linkmode=external -w -s'
popd
mv $GOPATH/src/github.com/usit-gd/nivlheim/server .

Expand Down Expand Up @@ -191,7 +197,6 @@ rm -rf %{buildroot}
%license LICENSE.txt
%doc README.md
%{_sbindir}/nivlheim_client
%config %{_sysconfdir}/nivlheim/version
%config(noreplace) %{_sysconfdir}/nivlheim/client.conf
%config %{_sysconfdir}/cron.d/nivlheim_client

Expand Down
51 changes: 36 additions & 15 deletions server/cgi/processarchive
Original file line number Diff line number Diff line change
Expand Up @@ -42,36 +42,46 @@ if ($@) {
$logger = Log::Log4perl->get_logger("syslogger");
}

# Connect to database
my %attr = ("AutoCommit" => 0);
my $dbh = DBI->connect("dbi:Pg:", "", "", \%attr);
if (!$dbh) {
print "Status: 500\nContent-Type: text/plain\n\n";
print "Unable to connect to Postgres database:\n";
print $DBI::errstr . "\n";
$logger->error("Unable to connect to Postgres: " . $DBI::errstr);
exit 1;
}

# Check parameters
my %params = parse_query_string();
unless (exists $params{'file'}) {
print "Status: 400\n\nMissing parameter: file\n";
$logger->debug("Missing parameter: file");
exit;
}
if ($params{'file'} =~ m![\\/]!) {
print "Status: 403\n\n";
$logger->debug("File path contains slashes; not allowed");
exit;
}
my $archivefile = $params{'file'};
$archivefile = "$confdir/queue/$archivefile";
unless (-r $archivefile) {
unless (-f $archivefile) {
print "Status: 410\n\nCan't find the archive file.\n";
$logger->debug("Can't find the archive file");
exit;
}
unless (-r $archivefile) {
print "Status: 400\n\nUnable to read the archive file.\n";
$logger->debug("Unable to read the archive file");
exit;
}

# Connect to database
$logger->debug("Connecting to database");
my %attr = ("AutoCommit" => 0);
my $dbh = DBI->connect("dbi:Pg:", "", "", \%attr);
if (!$dbh) {
print "Status: 500\nContent-Type: text/plain\n\n";
print "Unable to connect to Postgres database:\n";
print $DBI::errstr . "\n";
$logger->error("Unable to connect to Postgres: " . $DBI::errstr);
exit 1;
}

# Unpack the archive
eval{
$logger->debug("Create temp dir");
my $dir = tempdir( CLEANUP => 1 );
$logger->debug("Unpacking into $dir");
chdir($dir);
Expand All @@ -90,6 +100,7 @@ eval{
chmod(0755, "$dir/files", "$dir/commands");

# Text files from Windows are usually UTF-16, so let's convert those
$logger->debug("zip file, check files for UTF-16 and convert where applicable");
if ($archivefile =~ /\.zip$/) {
my $zip = Archive::Zip->new();
unless ($zip->read($archivefile) == AZ_OK) {
Expand Down Expand Up @@ -122,6 +133,7 @@ eval{
unlink("$dir/files/var/log/*");

# read the meta data file
$logger->debug("Read meta file");
my %meta = ();
open(my $F, "$archivefile.meta") || die "Unable to open metafile";
while (<$F>) {
Expand All @@ -133,6 +145,8 @@ eval{

my $iso_received = uts_to_rfc3339($meta{'received'});

$logger->debug("Prepare database statements");

my $currentfiles = $dbh->selectall_hashref("SELECT fileid,filename FROM files "
."WHERE certfp=? AND current", "filename", undef, ($meta{certfp}));

Expand Down Expand Up @@ -255,29 +269,34 @@ eval{
if ($dbh->err) { die $dbh->errstr; }
}
find(\&callback1, $dir);
$logger->debug("Completed inserting new files into the database");

# clear the "current" flag for files that weren't in this package
$logger->debug("Clear the current flag for deleted files");
foreach (keys %{$currentfiles}) {
my $fileid = $currentfiles->{$_}->{'fileid'};
$sth_clearcurrent->execute(($fileid));
push @no_longer_current, $fileid;
}

$logger->debug("Commit database transaction");
$dbh->commit;
unlink($archivefile, "$archivefile.meta");

# Create a task that will call the API to signal that these files are
# no longer current, so they can be removed from the search cache.
$logger->debug("Call API 'unsetCurrent'");
if ($#no_longer_current>-1) {
get("http://localhost:4040/api/internal/unsetCurrent?ids="
. join(',', @no_longer_current));
}

# Report the number of unchanged files to the system service
# so they too are counted (for statistical purposes)
$logger->debug("Call API 'countFiles'");
if ($unchangedfilecount > 0) {
get("http://localhost:4040/api/internal/countFiles?n=".$unchangedfilecount);
}

$dbh->commit;
unlink($archivefile, "$archivefile.meta");
};
if ($@) {
$logger->error($@);
Expand All @@ -291,6 +310,8 @@ if ($@) {
exit 1;
};

$logger->debug("Finished.");

# Clean up
$dbh->disconnect;

Expand Down
5 changes: 5 additions & 0 deletions server/service/accessControl.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ func wrapRequireAdmin(h http.Handler) http.Handler {
// has authenticated, either through Oauth2 or an API key.
func wrapRequireAuth(h httpHandlerWithAccessProfile) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
//TODO let local auth plugin through. handle this in a better way
if isLocal(req) {
h.ServeHTTP(w, req, &AccessProfile{isAdmin: true})
return
}
// If authentication is not enabled in config, let the request through
if !authRequired {
h.ServeHTTP(w, req, &AccessProfile{isAdmin: true})
Expand Down
71 changes: 2 additions & 69 deletions server/service/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ import (
"log"
"net/http"
"net/url"
"reflect"
"regexp"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -54,6 +52,8 @@ func createAPImuxer(theDB *sql.DB, devmode bool) *http.ServeMux {
wrapRequireAdmin(&apiMethodSettings{db: theDB}))
api.Handle("/api/v0/settings",
wrapRequireAdmin(&apiMethodSettings{db: theDB}))
api.Handle("/api/v0/resetWaitingTimeForFailedTasks",
wrapRequireAdmin(&apiMethodResetWaitingTime{db: theDB}))

// API functions that don't require authentication
api.Handle("/api/v0/status", &apiMethodStatus{db: theDB})
Expand Down Expand Up @@ -322,73 +322,6 @@ func contains(needle string, haystack []string) bool {
return false
}

// runJob sets the "trigger" flag on the Job struct in the jobs array,
// but it doesn't actually execute the job. The main loop does that.
func runJob(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
match := regexp.MustCompile("/(\\w+)$").FindStringSubmatch(req.URL.Path)
if match == nil {
http.Error(w, "Missing job name in URL path", http.StatusUnprocessableEntity)
return
}
for i, jobitem := range jobs {
t := reflect.TypeOf(jobitem.job)
if t.Name() == match[1] {
// this will make main run the job
jobs[i].trigger = true
http.Error(w, "OK", http.StatusNoContent)
return
}
}
http.Error(w, "Job not found.", http.StatusNotFound)
}

// unsetCurrent is an internal API function that the CGI scripts use
// to notify the system service/daemon that some file(s) have had
// their "current" flag cleared, and can be removed from the
// in-memory search cache.
func unsetCurrent(w http.ResponseWriter, req *http.Request) {
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
for _, s := range strings.Split(req.FormValue("ids"), ",") {
fileID, err := strconv.ParseInt(s, 10, 64)
if err == nil {
removeFileFromFastSearch(fileID)
}
}
http.Error(w, "OK", http.StatusNoContent)
}

// countFiles is an internal API function that the CGI scripts use
// to notify the system service/daemon that a number of files
// have been processed, so we can produce an accurate count of
// files-per-minute.
func countFiles(w http.ResponseWriter, req *http.Request) {
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
i, err := strconv.Atoi(req.FormValue("n"))
if err != nil || i == 0 {
return
}
pfib.Add(float64(i)) // pfib = parsed files interval buffer
}

func doNothing(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "無\n\n") // https://en.wikipedia.org/wiki/Mu_(negative)
}

func isTrueish(s string) bool {
s = strings.ToLower(s)
return s == "1" || s == "t" || s == "true" || s == "yes" || s == "y"
Expand Down
80 changes: 80 additions & 0 deletions server/service/api_internal.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package main

import (
"fmt"
"net/http"
"reflect"
"regexp"
"strconv"
"strings"
)

// runJob sets the "trigger" flag on the Job struct in the jobs array,
// but it doesn't actually execute the job. The main loop does that.
func runJob(w http.ResponseWriter, req *http.Request) {
if req.Method != "POST" {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
match := regexp.MustCompile("/(\\w+)$").FindStringSubmatch(req.URL.Path)
if match == nil {
http.Error(w, "Missing job name in URL path", http.StatusUnprocessableEntity)
return
}
for i, jobitem := range jobs {
t := reflect.TypeOf(jobitem.job)
if t.Name() == match[1] {
// this will make main run the job
jobs[i].trigger = true
http.Error(w, "OK", http.StatusNoContent)
return
}
}
http.Error(w, "Job not found.", http.StatusNotFound)
}

// unsetCurrent is an internal API function that the CGI scripts use
// to notify the system service/daemon that some file(s) have had
// their "current" flag cleared, and can be removed from the
// in-memory search cache.
func unsetCurrent(w http.ResponseWriter, req *http.Request) {
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
for _, s := range strings.Split(req.FormValue("ids"), ",") {
fileID, err := strconv.ParseInt(s, 10, 64)
if err == nil {
removeFileFromFastSearch(fileID)
}
}
http.Error(w, "OK", http.StatusNoContent)
}

// countFiles is an internal API function that the CGI scripts use
// to notify the system service/daemon that a number of files
// have been processed, so we can produce an accurate count of
// files-per-minute.
func countFiles(w http.ResponseWriter, req *http.Request) {
if !isLocal(req) {
http.Error(w, "Only local requests are allowed", http.StatusForbidden)
return
}
i, err := strconv.Atoi(req.FormValue("n"))
if err != nil || i == 0 {
return
}
pfib.Add(float64(i)) // pfib = parsed files interval buffer
}

func doNothing(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "無\n\n") // https://en.wikipedia.org/wiki/Mu_(negative)
//for k, v := range req.Header {
// fmt.Fprintln(w, k+" = "+strings.Join(v, ", "))
//}
}
6 changes: 5 additions & 1 deletion server/service/api_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@ func (vars *apiMethodStatus) ServeHTTP(w http.ResponseWriter, req *http.Request)
vars.db.QueryRow("SELECT count(*) FROM hostinfo").Scan(&status.NumOfMachines)

// NumOfFiles
vars.db.QueryRow("SELECT count(*) FROM files WHERE current").Scan(&status.NumOfFiles)
status.NumOfFiles = numberOfFilesInFastSearch()
if status.NumOfFiles == -1 {
// Slower method
vars.db.QueryRow("SELECT count(*) FROM files WHERE current").Scan(&status.NumOfFiles)
}

// ReportingPercentageLastHour
if status.NumOfMachines > 0 {
Expand Down
Loading

0 comments on commit a27b432

Please sign in to comment.