Merge pull request #10 from spiegel-im-spiegel/another-imprement

Another imprement

Gopkg.toml

@@ -20,17 +20,6 @@
 #  name = "github.com/x/y"
 #  version = "2.4.0"
 
-ignored = [
-  "github.com/cpuguy83/go-md2man/md2man",
-  "github.com/mitchellh/go-homedir",
-  "github.com/russross/blackfriday",
-  "github.com/spf13/cobra/cobra",
-  "github.com/spf13/cobra/cobra/cmd",
-  "github.com/spf13/cobra/doc",
-  "github.com/spf13/viper",
-  "gopkg.in/yaml.v2",
-]
-
 [[constraint]]
   name = "github.com/spf13/cobra"
   version = "~0.0.1"
@@ -39,10 +28,6 @@ ignored = [
   name = "github.com/spiegel-im-spiegel/gocli"
   version = "~0.4.0"
 
-[[constraint]]
-  name = "github.com/spiegel-im-spiegel/text"
-  version = "~0.1.0"
-
 [[constraint]]
   name = "github.com/pkg/errors"
   version = "~0.8.0"

mklink.go

@@ -1,14 +1,19 @@
 package mklink
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"strings"
 
+	"golang.org/x/net/html/charset"
+
 	"github.com/PuerkitoBio/goquery"
+	encoding "github.com/mattn/go-encoding"
 	"github.com/pkg/errors"
 )
 
@@ -23,31 +28,50 @@ type Link struct {
 //New returns new Link instance
 func New(url string) (*Link, error) {
 	link := &Link{URL: trimString(url)}
-	doc, err := goquery.NewDocument(link.URL)
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	link.Location = resp.Request.URL.String()
+
+	br := bufio.NewReader(resp.Body)
+	var r io.Reader = br
+	if data, err2 := br.Peek(1024); err2 == nil { //next 1024 bytes without advancing the reader.
+		enc, name, _ := charset.DetermineEncoding(data, resp.Header.Get("content-type"))
+		if enc != nil {
+			r = enc.NewDecoder().Reader(br)
+		} else if len(name) > 0 {
+			if enc := encoding.GetEncoding(name); enc != nil {
+				r = enc.NewDecoder().Reader(br)
+			}
+		}
+	}
+	doc, err := goquery.NewDocumentFromReader(r)
 	if err != nil {
 		return link, err
 	}
-	link.Location = doc.Url.String()
 
 	doc.Find("head").Each(func(_ int, s *goquery.Selection) {
 		s.Find("title").Each(func(_ int, s *goquery.Selection) {
-			t := ToUTF8([]byte(s.Text()))
+			t := s.Text()
 			if len(t) > 0 {
 				link.Title = trimString(t)
 			}
 		})
 		s.Find("meta[name='description']").Each(func(_ int, s *goquery.Selection) {
 			if v, ok := s.Attr("content"); ok {
-				d := ToUTF8([]byte(v))
-				if len(d) > 0 {
-					link.Description = trimString(d)
+				if len(v) > 0 {
+					link.Description = trimString(v)
 				}
 			}
 		})
 	})
 
 	return link, nil
 }
+
 func trimString(s string) string {
 	s = strings.Replace(s, "\n", " ", -1)
 	return strings.Trim(s, "\t ")