Remove panics bubble up errors + update (#1)

* fixes 0xrawsec#12 - only unmarshal UTF16String when the size is greater than zero

* Update evtx.File to support generic interfaces

* Docs

* Fix issue 0xrawsec#12

* Changed version in makefile

* Fixed issue 0xrawsec#16

* Corrected wrong fix of issue 0xrawsec#16, then also fix issue 0xrawsec#17

* Fixed issues: 0xrawsec#15 and 0xrawsec#21

* Version upgrade

* Version bump for evtxdump and evtxmon

* Fixed issue 0xrawsec#23

* Addressed issue 0xrawsec#25

* Fixed issue 0xrawsec#25

* Fixed issue 0xrawsec#27

* Standardized GoEvtxMap.Del API

* Fixed bug in GoEvtxMap.Del

* Return error from TemplateInstance.ElementToGoEvtx

* Bubble up error from NodeToGoEvtx

* Bubble up error from ElementToGoEvtx

* Bubble up errors from GoEvtxMap

* Update tests

* Panic on error in tests

Co-authored-by: Josh VanderLinden <[email protected]>
Co-authored-by: Quentin JEROME <[email protected]>

evtx/event.go

@@ -64,8 +64,8 @@ func (e Event) GoEvtxMap(c *Chunk) (pge *GoEvtxMap, err error) {
 	// Bug here if we put c
 	element, err := Parse(reader, c, false)
 	if err != nil && err != io.EOF {
-		//panic(err)
 		log.Error(err)
+		return nil, err
 	}
 	// If not a BinXMLFragment a panic will be raised
 	fragment, ok := element.(*Fragment)
@@ -76,7 +76,7 @@ func (e Event) GoEvtxMap(c *Chunk) (pge *GoEvtxMap, err error) {
 		// Way to raise panic
 		_ = element.(*Fragment)
 	}
-	return fragment.GoEvtxMap(), err
+	return fragment.GoEvtxMap()
 }
 
 func (e Event) String() string {

evtx/evtx.go

@@ -1,11 +1,13 @@
 package evtx
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
 	"io"
 	"math"
 	"os"
+	"regexp"
 	"sync"
 	"time"
 
@@ -42,6 +44,12 @@ func (cs ChunkSorter) Swap(i, j int) {
 
 //////////////////////////////////// File //////////////////////////////////////
 
+var (
+	ErrCorruptedHeader = fmt.Errorf("Corrupted header")
+	ErrDirtyFile       = fmt.Errorf("File is flagged as dirty")
+	ErrRepairFailed    = fmt.Errorf("File header could not be repaired")
+)
+
 // FileHeader structure definition
 type FileHeader struct {
 	Magic           [8]byte
@@ -58,30 +66,84 @@ type FileHeader struct {
 	CheckSum        uint32
 }
 
+func (f *FileHeader) Verify() error {
+	if !bytes.Equal(f.Magic[:], []byte("ElfFile\x00")) {
+		return ErrCorruptedHeader
+	}
+	// File is dirty
+	if f.Flags == 1 {
+		return ErrDirtyFile
+	}
+	return nil
+}
+
+// Repair the header. It makes sense to use this function
+// whenever the file is flagged as dirty
+func (f *FileHeader) Repair(r io.ReadSeeker) error {
+	chunkHeaderRE := regexp.MustCompile(ChunkMagic)
+	rr := bufio.NewReader(r)
+	cc := uint16(0)
+	for loc := chunkHeaderRE.FindReaderIndex(rr); loc != nil; loc = chunkHeaderRE.FindReaderIndex(rr) {
+		cc++
+	}
+
+	if f.ChunkCount > cc {
+		return ErrRepairFailed
+	}
+
+	// Fixing chunk count
+	f.ChunkCount = cc
+	// Fixing LastChunkNum
+	f.LastChunkNum = uint64(f.ChunkCount - 1)
+	// File is not dirty anymore
+	f.Flags = 0
+	return nil
+}
+
 // File structure definition
 type File struct {
 	sync.Mutex      // We need it if we want to parse (read) chunks in several threads
 	Header          FileHeader
-	file            *os.File
+	file            io.ReadSeeker
 	monitorExisting bool
 }
 
+// New EvtxFile structure initialized from an open buffer
+// @r : buffer containing evtx data to parse
+// return File : File structure initialized
+func New(r io.ReadSeeker) (ef File, err error) {
+	ef.file = r
+	ef.ParseFileHeader()
+	return
+}
+
 // New EvtxFile structure initialized from file
 // @filepath : filepath of the evtx file to parse
 // return File : File structure initialized
-func New(filepath string) (ef File, err error) {
+func Open(filepath string) (ef File, err error) {
 	file, err := os.Open(filepath)
 	if err != nil {
 		return
 	}
-	ef.file = file
-	ef.ParseFileHeader()
+
+	ef, err = New(file)
+	if err != nil {
+		return
+	}
+
+	err = ef.Header.Verify()
+
 	return
 }
 
-// Open alias to New to be complient with the Go way of programming
-func Open(filepath string) (ef File, err error) {
-	return New(filepath)
+// OpenDirty is a wrapper around Open to handle the case
+// where the file opened has its dirty flag set
+func OpenDirty(filepath string) (ef File, err error) {
+	// Repair the file header if file is dirty
+	if ef, err = Open(filepath); err == ErrDirtyFile {
+		err = ef.Header.Repair(ef.file)
+	}
+	return
 }
 
 // SetMonitorExisting sets monitorExisting flag of EvtxFile struct in order to
@@ -95,6 +157,7 @@ func (ef *File) SetMonitorExisting(value bool) {
 func (ef *File) ParseFileHeader() {
 	ef.Lock()
 	defer ef.Unlock()
+
 	GoToSeeker(ef.file, 0)
 	err := encoding.Unmarshal(ef.file, &ef.Header, Endianness)
 	if err != nil {
@@ -104,7 +167,7 @@ func (ef *File) ParseFileHeader() {
 
 func (fh FileHeader) String() string {
 	return fmt.Sprintf(
-		"Magic: %s\n"+
+		"Magic: %q\n"+
 			"FirstChunkNum: %d\n"+
 			"LastChunkNum: %d\n"+
 			"NumNextRecord: %d\n"+
@@ -176,8 +239,6 @@ func (ef *File) FetchChunk(offset int64) (Chunk, error) {
 
 // Chunks returns a chan of all the Chunks found in the current file
 // return (chan Chunk)
-// TODO: need to be improved: the chunk do not need to be loaded into memory there
-// we just need the header to sort them out. If we do so, do not need undordered chunks
 func (ef *File) Chunks() (cc chan Chunk) {
 	ss := datastructs.NewSortedSlice(0, int(ef.Header.ChunkCount))
 	cc = make(chan Chunk)
@@ -310,10 +371,17 @@ func (ef *File) Events() (cgem chan *GoEvtxMap) {
 	go func() {
 		defer close(cgem)
 		for c := range ef.Chunks() {
-			for e := range c.Events() {
-				cgem <- e
+			cpc, err := ef.FetchChunk(c.Offset)
+			switch {
+			case err != nil && err != io.EOF:
+				panic(err)
+			case err == nil:
+				for ev := range cpc.Events() {
+					cgem <- ev
+				}
 			}
 		}
+
 	}()
 	return
 }
@@ -329,10 +397,6 @@ func (ef *File) FastEvents() (cgem chan *GoEvtxMap) {
 		go func() {
 			defer close(chanQueue)
 			for pc := range ef.Chunks() {
-				// We have to create a copy here because otherwise cpc.EventsChan() fails
-				// I guess that because EventsChan takes a pointer to an object and that
-				// and thus the chan is taken on the pointer and since the object pointed
-				// changes -> kaboom
 				cpc, err := ef.FetchChunk(pc.Offset)
 				switch {
 				case err != nil && err != io.EOF:
@@ -429,5 +493,9 @@ func (ef *File) MonitorEvents(stop chan bool, sleep ...time.Duration) (cgem chan
 
 // Close file
 func (ef *File) Close() error {
-	return ef.file.Close()
+	if f, ok := ef.file.(io.Closer); ok {
+		return f.Close()
+	}
+
+	return nil
 }

evtx/globals.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"math"
 	"runtime"
+	"sync"
 	"time"
 )
 
@@ -23,6 +24,8 @@ var (
 
 //////////////////////// Global Variables and their setters /////////////////////
 var (
+	// Debug mode for parser
+	Debug = false
 	// ModeCarving flag to identify we run in carving mode
 	ModeCarving = false
 	// DefaultMonitorSleep default sleep time between two file update checks when
@@ -43,10 +46,16 @@ func SetMonitorSleep(d time.Duration) {
 	DefaultMonitorSleep = d
 }
 
+// SetMaxJobs sets the number of jobs for parsing
 func SetMaxJobs(jobs int) {
 	MaxJobs = jobs
 }
 
+// SetDebug set variable enabling debugging at parser level
+func SetDebug(value bool) {
+	Debug = value
+}
+
 ////////////////////////// EVTX Constants and globs ////////////////////////////
 
 const (
@@ -70,10 +79,16 @@ const (
 	MaxSliceSize = ChunkSize
 )
 
+//type LastParsedElements
+
 var (
 	Endianness = binary.LittleEndian
 	// Used for debug purposes
-	lastParsedElements [4]Element
+	//lastParsedElements LastParsedElements
+	lastParsedElements struct {
+		sync.RWMutex
+		elements [4]Element
+	}
 )
 
 //////////////////////////////// BinXMLTokens //////////////////////////////////

evtx/goevtx.go

@@ -191,7 +191,7 @@ func (pg *GoEvtxMap) GetInt(path *GoEvtxPath) (int64, error) {
 	if err != nil {
 		return 0, &ErrEvtxEltNotFound{*path}
 	}
-	i, err := strconv.ParseInt(s, 10, 64)
+	i, err := strconv.ParseInt(s, 0, 64)
 	if err != nil {
 		return 0, err
 	}
@@ -214,7 +214,7 @@ func (pg *GoEvtxMap) GetUint(path *GoEvtxPath) (uint64, error) {
 	if err != nil {
 		return 0, &ErrEvtxEltNotFound{*path}
 	}
-	u, err := strconv.ParseUint(s, 10, 64)
+	u, err := strconv.ParseUint(s, 0, 64)
 	if err != nil {
 		return 0, err
 	}
@@ -393,16 +393,22 @@ func (pg *GoEvtxMap) Set(path *GoEvtxPath, new GoEvtxElement) error {
 }
 
 // Del deletes the object referenced by path
-func (pg *GoEvtxMap) Del(path ...string) {
-	if len(path) > 0 {
-		if ge, ok := (*pg)[path[0]]; ok {
-			if len(path) == 1 {
-				delete((*pg), path[0])
+func (pg *GoEvtxMap) Del(path *GoEvtxPath) {
+	if len(*path) > 0 {
+		if ge, ok := (*pg)[(*path)[0]]; ok {
+			if len(*path) == 1 {
+				delete((*pg), (*path)[0])
 			}
 			switch ge.(type) {
 			case GoEvtxMap:
 				p := ge.(GoEvtxMap)
-				p.Del(path[1:]...)
+				np := (*path)[1:]
+				p.Del(&np)
+
+			case map[string]interface{}:
+				p := GoEvtxMap(ge.(map[string]interface{}))
+				np := (*path)[1:]
+				p.Del(&np)
 			}
 		}
 	}
@@ -411,5 +417,5 @@ func (pg *GoEvtxMap) Del(path ...string) {
 // DelXmlns : utility function to delete useless xlmns entry found in every
 // GoEvtxMap
 func (pg *GoEvtxMap) DelXmlns() {
-	pg.Del(XmlnsPath...)
+	pg.Del(&XmlnsPath)
 }

evtx/node.go

@@ -34,11 +34,11 @@ func NodeTree(es []Element, index int) (Node, int) {
 }
 
 // TODO: Not used
-func ElementToGoEvtx(elt Element) GoEvtxElement {
+func ElementToGoEvtx(elt Element) (GoEvtxElement, error) {
 	switch elt.(type) {
 	// BinXML specific
 	case *ValueText:
-		return elt.(*ValueText).String()
+		return elt.(*ValueText).String(), nil
 	/*case *OptionalSubstitution:
 		s := elt.(*OptionalSubstitution)
 		return ElementToGoEvtx(ti.Data.Values[int(s.SubID)])
@@ -55,8 +55,8 @@ func ElementToGoEvtx(elt Element) GoEvtxElement {
 		root := temp.Root()
 		return temp.NodeToGoEvtx(&root)
 	case Value:
-		return elt.(Value).Repr()
+		return elt.(Value).Repr(), nil
 	default:
-		panic(fmt.Errorf("Don't know how to handle: %T", elt))
+		return nil, fmt.Errorf("Don't know how to handle: %T", elt)
 	}
 }

evtx/parser.go

@@ -14,20 +14,23 @@ func checkParsingError(err error, reader io.ReadSeeker, e Element) {
 	UpdateLastElements(e)
 	if err != nil {
 		log.DontPanicf("%s: parsing %T", err, e)
-		DebugReader(reader, 10, 5)
+		if Debug {
+			DebugReader(reader, 10, 5)
+		}
 	}
 }
 
 func checkFullParsingError(err error, reader io.ReadSeeker, e Element, c *Chunk) {
 	UpdateLastElements(e)
 	if err != nil {
-		//log.DontPanicf("%s: parsing %T (chunk @ 0x%08x reader @ 0x%08x)", err, e, c.Offset, BackupSeeker(reader))
 		if c != nil {
 			log.DebugDontPanicf("%s: parsing %T (chunk @ 0x%08x reader @ 0x%08x)", err, e, c.Offset, BackupSeeker(reader))
 		} else {
 			log.DebugDontPanicf("%s: parsing %T (chunk @ NIL reader @ 0x%08x)", err, e, BackupSeeker(reader))
 		}
-		DebugReader(reader, 10, 5)
+		if Debug {
+			DebugReader(reader, 10, 5)
+		}
 	}
 }