Merge pull request #22 from voidshard/add_docker_support

v1.1

README.md

@@ -217,14 +217,11 @@ That is, this search will return all items from any collection(s) that have eith
 - item type of "b"
 
 
-## Viewer
-
-* Go & ReactJS: https://github.com/voidshard/muninn (simple read-only viewer used in development)
-
 ## Notes
 
-- There will be a delay between when something is created in wysteria, and when it is returnable via a search. This delay should be measured in seconds at most, depending on the 'searchbase' being used. It's possible that different searchbase implementations will overcome this in the future
-- "Moving" and/or "renaming" are foreign concepts to wysteria, if you need to move something you should delete and recreate it. Part of this is because of the complications it introduces, and part of this is to be able to support deterministic ids (in a later version). That is, most objects and fields are immutable after creation.
+- Unless you are searching by Id, there will be a delay between when something is created in wysteria, and when it is returnable via a search. This delay should be measured in seconds at most, depending on the 'searchbase' being used.
+    - There is a searchbase config 'ReindexOnWrite' to mitigate this but this is expensive to do: Using this should be considered very carefully.
+- "Moving" and/or "renaming" are foreign concepts to wysteria, if you need to move something you should delete and recreate it (or enforce some kind of naming scheme so you don't screw up names..). That is, most objects and fields are immutable after creation.
 - Wysteria makes no attempt to understand the links you set or sanitize strings given. The meaning behind the resource 'location' for example is left entirely to the user -- it could be a filepath, a url, an id into another system or something else entirely.
 - There is no maximum number of facets, links or resources that one can attach that is enforced by the system. Practical limits of the underlying systems will begin to apply at some point however.
 
@@ -236,14 +233,41 @@ Also, if (when) you find bugs, let me know.
 
 
 ## ToDo List
-- more unittests for business logic
 - admin console 
   - extend the wysteria server chan to allow realtime management of live server(s)
     - allow / disallow certain client requests
     - introduce some kind of configurable auto load-balancing 
     - allow changing of configuration option(s) 
     - add alerts for certain events or server statuses
     - allow temporary rerouting of client requests
-- implement system for deterministic ids
-- ability to write to backup db / sb (might be useful for hot swapping or dev dataset)
-- ability to swap over to backup db without server restart
+- viewer
+
+## v1.1
+- Elasticsearch v2 support dropped from master
+- Elasticsearch v6 (latest) support added
+- Searching for an object(s) by Id(s) bypasses the searchbase, meaning ID searches are faster & have no delay.
+- Created IDs are now deterministic, and are the same regardless of the database backend used.
+- monitor that writes to stdout now writes time taken in call in millis to be more human readable.
+- Docker & docker compose support for
+    - local builds (compiled using the local wysteria src)
+    - master builds (compiled by cloning github repo)
+    - multi container (nats-elastic-mongo) & all in one (gRPC-bleve-bolt) builds
+- Integration test suite added using all of the above docker builds
+- gRPC fixes & timeout improvements
+- SSL support added to gRPC middleware
+- added some sugar to set a 'FacetLinkType' on a Link obj when created - it's either FacetVersionLink or FacetItemLink
+- clientside now sets facets on local objects on a SetFacets() call(s)
+- Removed random extra SetFacets() call when a collection is created
+- Fixed a few misc nil pointer fixes that could occur on the server
+- Removal of cascading deletes -> only links & resources are deleted when their parent goes
+    - The server should now block deletes if it has children (links & resources do not apply to this)
+- Delete calls no longer spawn extra routines
+- Renamed lots of funcs & vars so they fit the Go style
+- more doc strings
+- links are now returned when LinkTo is called. The return signature of item and version .LinkTo is now (*Link, error)
+- resources are now returned when AddResource is called. Return signature of version.AddResource is (*Resource, error)
+- increased monitoring. There is now monitoring around middleware (enter / exit), database and searchbase calls
+- resource uniqueness is enforced for a given location, name, type & parent
+- Added config var 'ReindexOnWrite' that ensures that written data is immediately searchable. This is useful in tests
+  where we want to immediately fetch things and the performance hit isn't considered a problem.
+  Setting this to true for any prod system should be considered carefully as it's expected to be very costly.

client/client.go

@@ -8,14 +8,24 @@ to communicate.
 package wysteria_client
 
 import (
+	"errors"
+	"fmt"
 	wyc "github.com/voidshard/wysteria/common"
 	wcm "github.com/voidshard/wysteria/common/middleware"
-	"fmt"
-	"errors"
 )
 
 const (
 	defaultSearchLimit = 500
+
+	// These are copied over from github.com/voidshard/wysteria/common for convenience
+	//
+	FacetRootCollection = "/"
+	FacetCollection     = "collection"
+	FacetItemType       = "itemtype"
+	FacetItemVariant    = "variant"
+	FacetLinkType       = "linktype"
+	FacetVersionLink    = "version"
+	FacetItemLink       = "item"
 )
 
 // Client wraps the desired middleware and supplies a more user friendly interface to users
@@ -60,6 +70,39 @@ func Driver(name string) ClientOption {
 	}
 }
 
+// Set SSL certificate to use
+//
+func SSLCert(in string) ClientOption {
+	return func(i *Client) {
+		i.settings.Middleware.SSLCert = in
+	}
+}
+
+// Set SSL key to use
+//
+func SSLKey(in string) ClientOption {
+	return func(i *Client) {
+		i.settings.Middleware.SSLKey = in
+	}
+}
+
+// Verify SSL certificates on connecting
+//  That is, if this is 'false' we'll accept insecure certificates (like self signed certs for example)
+//
+func SSLVerify(in bool) ClientOption {
+	return func(i *Client) {
+		i.settings.Middleware.SSLVerify = in
+	}
+}
+
+// Enable SSL
+//
+func SSLEnable(in bool) ClientOption {
+	return func(i *Client) {
+		i.settings.Middleware.SSLEnableTLS = in
+	}
+}
+
 // Create a new client and connect to the server
 func New(opts ...ClientOption) (*Client, error) {
 	client := &Client{
@@ -117,12 +160,12 @@ type clientStruct interface {
 	initUserFacets(map[string]string)
 }
 
-// Set the inital Facets on the soon to be created child
+// Set the initial Facets on the soon to be created child
 //  Note that you still cannot overwrite client reserved facets this way
 func Facets(in map[string]string) CreateOption {
 	return func(parent, child clientStruct) {
 		if in != nil {
 			child.initUserFacets(in)
 		}
 	}
-}
+}

client/collection.go

@@ -27,7 +27,11 @@ func (c *Collection) ParentId() string {
 
 // Return the parent collection of this collection (if any)
 func (c *Collection) Parent() (*Collection, error) {
-	return c.conn.Collection(c.ParentId())
+	found, err := c.conn.Search(Id(c.ParentId())).FindCollections(Limit(1))
+	if err != nil || len(found) != 1 {
+		return nil, err
+	}
+	return found[0], err
 }
 
 // Get the facet value and a bool indicating if the value exists for the given key.
@@ -43,6 +47,12 @@ func (i *Collection) Facets() map[string]string {
 
 // Set all the key:value pairs given on this Collection's facets.
 func (i *Collection) SetFacets(in map[string]string) error {
+	if in == nil {
+		return nil
+	}
+	for k, v := range in {
+		i.data.Facets[k] = v
+	}
 	return i.conn.middleware.UpdateCollectionFacets(i.data.Id, in)
 }
 
@@ -110,8 +120,7 @@ func (w *Client) createCollection(name string, parent *Collection, opts ...Creat
 	for _, opt := range opts {
 		opt(parent, child)
 	}
-
-	if parent == nil { // Nb this will overwrite facets set by users that we're going to use -> this is intentional
+	if parent == nil {
 		col.Facets[wyc.FacetCollection] = wyc.FacetRootCollection
 	} else {
 		col.Parent = parent.Id()
@@ -123,16 +132,16 @@ func (w *Client) createCollection(name string, parent *Collection, opts ...Creat
 		return nil, err
 	}
 	col.Id = collection_id
-	return child, child.SetFacets(child.Facets())
+	return child, nil
 }
 
 // Create a new collection & return it (that is, a collection with no parent)
 //  - The collection name is required to be unique among all collections
 func (w *Client) CreateCollection(name string, opts ...CreateOption) (*Collection, error) {
-	return w.createCollection(name,  nil, opts...)
+	return w.createCollection(name, nil, opts...)
 }
 
 // Set initial user defined facets
 func (c *Collection) initUserFacets(in map[string]string) {
 	c.data.Facets = in
-}
+}

client/config.go

@@ -21,12 +21,12 @@ type configuration struct {
 func init() {
 	config = makeDefaults()
 
-	config_filepath, err := common.ChooseClientConfig()
+	configFilepath, err := common.ChooseClientConfig()
 	if err == nil {
 		cnf := &configuration{}
-		err := common.ReadConfig(config_filepath, cnf)
+		err := common.ReadConfig(configFilepath, cnf)
 		if err != nil {
-			log.Println("Unable to read config", config_filepath, err)
+			log.Println("Unable to read config", configFilepath, err)
 		} else {
 			config = cnf
 		}

client/examples/01/main.go

@@ -35,7 +35,7 @@ func main() {
 	}
 
 	// Add some kind of resource(s) to our tree
-	err = oak01.AddResource("default", "png", "url://images/oak01.png")
+	_, err = oak01.AddResource("default", "png", "url://images/oak01.png")
 	if err != nil {
 		panic(err)
 	}
@@ -81,7 +81,7 @@ func main() {
 	forest_map_01.Publish()
 
 	// Now, let's create links on our forest to it's constituent trees
-	err = forest_map_01.LinkTo("oak", oak02, wysteria.Facets(customFacets))
+	_, err = forest_map_01.LinkTo("oak", oak02, wysteria.Facets(customFacets))
 	if err != nil {
 		panic(err)
 	}

client/examples/07/main.go

@@ -27,34 +27,34 @@ func main() {
 
 	// Retrieve the current published version (we linked the Version, but we could have linked
 	// items in the same manner
-	forest_01, err := items[0].PublishedVersion()
+	forest01, err := items[0].PublishedVersion()
 	if err != nil {
 		panic(err)
 	}
 
 	// Retrieve all linked versions
-	linked_versions, err := forest_01.Linked()
+	linkedVersions, err := forest01.Linked()
 	if err != nil {
 		panic(err)
 	}
 
-	for link_name, found_versions := range linked_versions {
-		for _, version := range found_versions {
-			fmt.Println(link_name, version.Version())
+	for linkName, foundVersions := range linkedVersions {
+		for _, version := range foundVersions {
+			fmt.Println(linkName, version.Version())
 		}
 	}
 	//elm 1
 	//pine 1
 	//oak 2
 
 	// We can also grab links with a specific name
-	desired_link_name := "elm"
-	linked, err := forest_01.Linked(wysteria.Name(desired_link_name))
+	desiredLinkName := "elm"
+	linked, err := forest01.Linked(wysteria.Name(desiredLinkName))
 	if err != nil {
 		panic(err)
 	}
-	for _, version := range linked[desired_link_name] {
-		fmt.Println(desired_link_name, version.Version())
+	for _, version := range linked[desiredLinkName] {
+		fmt.Println(desiredLinkName, version.Version())
 	}
 	//elm 1
 }

client/examples/08/main.go

@@ -15,42 +15,42 @@ func main() {
 
 	// Let's get our published pine
 	items, _ := client.Search(wysteria.ItemType("tree"), wysteria.ItemVariant("pine")).FindItems()
-	published_version, _ := items[0].PublishedVersion()
+	publishedVersion, _ := items[0].PublishedVersion()
 
 	// We can grab resources by Name
-	default_resources, err := published_version.Resources(wysteria.Name("default"))
+	defaultResources, err := publishedVersion.Resources(wysteria.Name("default"))
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println("--default resources--")
-	for _, resource := range default_resources {
+	for _, resource := range defaultResources {
 		fmt.Println(resource.Name(), resource.Type(), resource.Location())
 	}
 	//--default resources--
 	//default png /path/to/pine01.png
 
 	// by Type
-	json_resources, _ := published_version.Resources(wysteria.ResourceType("json"))
+	jsonResources, _ := publishedVersion.Resources(wysteria.ResourceType("json"))
 	fmt.Println("--json resources--")
-	for _, resource := range json_resources {
+	for _, resource := range jsonResources {
 		fmt.Println(resource.Name(), resource.Type(), resource.Location())
 	}
 	//--json resources--
 	//stats json url://file.json
 
 	// some combination of the two
-	lowres_jpgs, _ := published_version.Resources(wysteria.Name("lowres"), wysteria.ResourceType("jpeg"))
+	lowresJpgs, _ := publishedVersion.Resources(wysteria.Name("lowres"), wysteria.ResourceType("jpeg"))
 	fmt.Println("--lowres jpeg resources--")
-	for _, resource := range lowres_jpgs {
+	for _, resource := range lowresJpgs {
 		fmt.Println(resource.Name(), resource.Type(), resource.Location())
 	}
 	//--lowres jpeg resources--
 	//lowres jpeg /path/lowres/image.jpeg
 
 	// Or grab all of them
-	all_resources, _ := published_version.Resources()
+	allResources, _ := publishedVersion.Resources()
 	fmt.Println("--resources--")
-	for _, resource := range all_resources {
+	for _, resource := range allResources {
 		fmt.Println(resource.Name(), resource.Type(), resource.Location())
 	}
 	//--resources--

client/examples/10/main.go

@@ -25,11 +25,22 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	log.Println("Delete maps:", collection.Delete())
 
-	collection, err = client.Collection("tiles")
+	err = collection.Delete()
+	if err == nil {
+		panic(err)
+	} else {
+		log.Println("[expected] Can't delete maps:", err)
+	}
+
+	anotherCollection, err := client.CreateCollection("thisisarandomstring")
+	if err != nil {
+		panic(err)
+	}
+	err = anotherCollection.Delete()
 	if err != nil {
 		panic(err)
+	} else {
+		log.Println("Deleted!")
 	}
-	log.Println("Delete tiles:", collection.Delete())
 }