embed.go

/*
Package embed allows for storing data resources in a virtual filesystem
that gets compiled directly into the output program, eliminating the need
to distribute data files with the application. This is especially useful for small
web servers that need to deliver content files.

The data is gzipped to save space.

An external tool for generating output files can be found at http://github.com/cratonica/embedder

Author: Clint Caywood

http://github.com/cratonica/embed
*/
package embed

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// A mapping of resource identifiers to associated data
type ResourceMap map[string][]byte

// A compressed resource map that can be used for serialization
type PackedResourceMap []byte

var byteOrder binary.ByteOrder = binary.LittleEndian

// Packs the map of resource identifiers => []byte into
// a buffer. This process is reversed by calling Unpack.
func Pack(data ResourceMap) (PackedResourceMap, error) {
	var buf bytes.Buffer
	writer, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
	if err != nil {
		return nil, err
	}
	keys := make([]string, len(data))
	idx := 0
	for key := range data {
		keys[idx] = key
		idx++
	}
	sort.Strings(keys)
	for _, i := range keys {
		v := data[i]
		if err = binary.Write(writer, byteOrder, int32(len(i))); err != nil {
			return nil, err
		}
		if _, err = writer.Write([]byte(i)); err != nil {
			return nil, err
		}
		if err = binary.Write(writer, byteOrder, int32(len(v))); err != nil {
			return nil, err
		}
		if _, err = writer.Write(v); err != nil {
		}
	}
	if err = writer.Close(); err != nil {
		return nil, err
	}
	result := buf.Bytes()
	return result, nil
}

type DeserializationError struct {
	what     string
	expected int
	actual   int
}

func (this *DeserializationError) Error() string {
	return fmt.Sprintf("Failure deserializing resource: expected %v to be %v bytes, but only read %v bytes", this.what, this.expected, this.actual)
}

func NewDeserializationError(what string, expected int, actual int) *DeserializationError {
	return &DeserializationError{what, expected, actual}
}

// Reads a buffer generated by a call to Pack, returning the original map
func Unpack(data PackedResourceMap) (ResourceMap, error) {
	result := make(map[string][]byte)
	buf := bytes.NewBuffer(data)
	reader, err := gzip.NewReader(buf)
	defer reader.Close()
	if err != nil {
		return nil, err
	}
	for {
		var size int32
		err := binary.Read(reader, byteOrder, &size)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, err
		}
		keyBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		err = binary.Read(reader, byteOrder, &size)
		if err != nil {
			return nil, err
		}
		dataBuf, err := readAll(reader, int(size))
		if err != nil {
			return nil, err
		}
		result[string(keyBuf)] = dataBuf
	}
	return result, nil
}

// Recursively packs all files in the given directory into
// a resource map. Directory delimiters are converted to Unix-style /
// regardless of the host operating system.
func CreateFromFiles(path string) (ResourceMap, error) {
	result := make(map[string][]byte)
	err := crawl(filepath.Clean(path), filepath.Clean(path), result)
	if err != nil {
		return nil, err
	}
	return result, nil
}

// Generates Go code containing the given data that can be included in the target project
func GenerateGoCode(packageName string, varName string, data PackedResourceMap) string {
	template := `
package %v

import "github.com/cratonica/embed"

var %v embed.PackedResourceMap = %#v
`
	return fmt.Sprintf(template, packageName, varName, data)
}

func readAll(reader io.Reader, size int) ([]byte, error) {
	result := make([]byte, size)
	totalBytes := 0
	for totalBytes < size {
		readSize, err := reader.Read(result[totalBytes:])
		if err != nil {
			if err == io.EOF && totalBytes+readSize == size {
				return result, nil 
			} else {
				return nil, err 
			}
		}
		totalBytes += readSize
	}
	return result, nil
}

func crawl(rootPath string, currentPath string, dest map[string][]byte) error {
	files, err := ioutil.ReadDir(currentPath)
	if err != nil {
		return err
	}
	for _, file := range files {
		fullPath := filepath.Join(currentPath, file.Name())
		if file.IsDir() {
			err := crawl(rootPath, fullPath, dest)
			if err != nil {
				return err
			}
		} else {
			buf, err := ioutil.ReadFile(fullPath)
			if err != nil {
				log.Printf("Failure reading file %v: %v", fullPath, err)
			} else {
				key := strings.TrimLeft(fullPath, rootPath)
				if strings.HasPrefix(key, string(os.PathSeparator)) {
					key = key[1:]
				}
				// Convert to unix style for internal use
				key = strings.Replace(key, string(os.PathSeparator), "/", -1)
				dest[key] = buf
			}
		}
	}
	return nil
}