Allow getJSON errors to be ignored

This change is mostly motivated to get a more stable CI build (we're building the Hugo site there, with Instagram and Twitter shortcodes sometimes failing).

Fixes gohugoio#7866

.travis.yml

@@ -72,6 +72,6 @@ script:
         HUGO_TIMEOUT=30000 mage -v check;
     fi
   - mage -v hugo
-  - ./hugo -s docs/
-  - ./hugo --renderToMemory -s docs/
+  - HUGO_IGNOREERRORS=error-remote-getjson ./hugo -s docs/
+  - HUGO_IGNOREERRORS=error-remote-getjson ./hugo --renderToMemory -s docs/
   - df -h

commands/commandeer.go

@@ -60,7 +60,7 @@ type commandeerHugoState struct {
 type commandeer struct {
 	*commandeerHugoState
 
-	logger       *loggers.Logger
+	logger       loggers.Logger
 	serverConfig *config.Server
 
 	// Currently only set when in "fast render mode". But it seems to
@@ -112,7 +112,7 @@ func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
 }
 
 func (c *commandeer) errCount() int {
-	return int(c.logger.ErrorCounter.Count())
+	return int(c.logger.LogCounters().ErrorCounter.Count())
 }
 
 func (c *commandeer) getErrorWithContext() interface{} {
@@ -415,7 +415,7 @@ func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 	}
 	config.Set("cacheDir", cacheDir)
 
-	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
+	cfg.Logger.Infoln("Using config file:", config.ConfigFileUsed())
 
 	return nil
 

commands/commands.go

@@ -322,16 +322,12 @@ func (cc *hugoBuilderCommon) handleFlags(cmd *cobra.Command) {
 	_ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"})
 }
 
-func checkErr(logger *loggers.Logger, err error, s ...string) {
+func checkErr(logger loggers.Logger, err error, s ...string) {
 	if err == nil {
 		return
 	}
-	if len(s) == 0 {
-		logger.CRITICAL.Println(err)
-		return
-	}
 	for _, message := range s {
-		logger.ERROR.Println(message)
+		logger.Errorln(message)
 	}
-	logger.ERROR.Println(err)
+	logger.Errorln(err)
 }

commands/convert.go

@@ -123,7 +123,7 @@ func (cc *convertCmd) convertContents(format metadecoders.Format) error {
 
 	site := h.Sites[0]
 
-	site.Log.FEEDBACK.Println("processing", len(site.AllPages()), "content files")
+	site.Log.Println("processing", len(site.AllPages()), "content files")
 	for _, p := range site.AllPages() {
 		if err := cc.convertAndSavePage(p, site, format); err != nil {
 			return err
@@ -147,19 +147,19 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target
 
 	errMsg := fmt.Errorf("Error processing file %q", p.Path())
 
-	site.Log.INFO.Println("Attempting to convert", p.File().Filename())
+	site.Log.Infoln("Attempting to convert", p.File().Filename())
 
 	f := p.File()
 	file, err := f.FileInfo().Meta().Open()
 	if err != nil {
-		site.Log.ERROR.Println(errMsg)
+		site.Log.Errorln(errMsg)
 		file.Close()
 		return nil
 	}
 
 	pf, err := pageparser.ParseFrontMatterAndContent(file)
 	if err != nil {
-		site.Log.ERROR.Println(errMsg)
+		site.Log.Errorln(errMsg)
 		file.Close()
 		return err
 	}
@@ -179,7 +179,7 @@ func (cc *convertCmd) convertAndSavePage(p page.Page, site *hugolib.Site, target
 	var newContent bytes.Buffer
 	err = parser.InterfaceToFrontMatter(pf.FrontMatter, targetFormat, &newContent)
 	if err != nil {
-		site.Log.ERROR.Println(errMsg)
+		site.Log.Errorln(errMsg)
 		return err
 	}
 

commands/hugo.go

@@ -130,7 +130,7 @@ func initializeConfig(mustHaveConfigFile, running bool,
 
 }
 
-func (c *commandeer) createLogger(cfg config.Provider, running bool) (*loggers.Logger, error) {
+func (c *commandeer) createLogger(cfg config.Provider, running bool) (loggers.Logger, error) {
 	var (
 		logHandle       = ioutil.Discard
 		logThreshold    = jww.LevelWarn
@@ -374,12 +374,12 @@ func (c *commandeer) initMemProfile() {
 
 	f, err := os.Create(c.h.memprofile)
 	if err != nil {
-		c.logger.ERROR.Println("could not create memory profile: ", err)
+		c.logger.Errorf("could not create memory profile: ", err)
 	}
 	defer f.Close()
 	runtime.GC() // get up-to-date statistics
 	if err := pprof.WriteHeapProfile(f); err != nil {
-		c.logger.ERROR.Println("could not write memory profile: ", err)
+		c.logger.Errorf("could not write memory profile: ", err)
 	}
 }
 
@@ -518,7 +518,7 @@ func (c *commandeer) build() error {
 		if createCounter, ok := c.destinationFs.(hugofs.DuplicatesReporter); ok {
 			dupes := createCounter.ReportDuplicates()
 			if dupes != "" {
-				c.logger.WARN.Println("Duplicate target paths:", dupes)
+				c.logger.Warnln("Duplicate target paths:", dupes)
 			}
 		}
 	}
@@ -532,8 +532,8 @@ func (c *commandeer) build() error {
 		baseWatchDir := c.Cfg.GetString("workingDir")
 		rootWatchDirs := getRootWatchDirsStr(baseWatchDir, watchDirs)
 
-		c.logger.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
-		c.logger.FEEDBACK.Println("Press Ctrl+C to stop")
+		c.logger.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
+		c.logger.Println("Press Ctrl+C to stop")
 		watcher, err := c.newWatcher(watchDirs...)
 		checkErr(c.Logger, err)
 		defer watcher.Close()
@@ -590,7 +590,7 @@ func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesy
 	staticFilesystems := c.hugo().BaseFs.SourceFilesystems.Static
 
 	if len(staticFilesystems) == 0 {
-		c.logger.INFO.Println("No static directories found to sync")
+		c.logger.Infoln("No static directories found to sync")
 		return langCount, nil
 	}
 
@@ -662,13 +662,13 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
 	syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")
 
 	if syncer.Delete {
-		c.logger.INFO.Println("removing all files from destination that don't exist in static dirs")
+		c.logger.Infoln("removing all files from destination that don't exist in static dirs")
 
 		syncer.DeleteFilter = func(f os.FileInfo) bool {
 			return f.IsDir() && strings.HasPrefix(f.Name(), ".")
 		}
 	}
-	c.logger.INFO.Println("syncing static files to", publishDir)
+	c.logger.Infoln("syncing static files to", publishDir)
 
 	// because we are using a baseFs (to get the union right).
 	// set sync src to root
@@ -689,7 +689,7 @@ func (c *commandeer) firstPathSpec() *helpers.PathSpec {
 
 func (c *commandeer) timeTrack(start time.Time, name string) {
 	elapsed := time.Since(start)
-	c.logger.FEEDBACK.Printf("%s in %v ms", name, int(1000*elapsed.Seconds()))
+	c.logger.Printf("%s in %v ms", name, int(1000*elapsed.Seconds()))
 }
 
 // getDirList provides NewWatcher() with a list of directories to watch for changes.
@@ -698,7 +698,7 @@ func (c *commandeer) getDirList() ([]string, error) {
 
 	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
 		if err != nil {
-			c.logger.ERROR.Println("walker: ", err)
+			c.logger.Errorln("walker: ", err)
 			return nil
 		}
 
@@ -724,7 +724,7 @@ func (c *commandeer) getDirList() ([]string, error) {
 
 		w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: fi, WalkFn: walkFn})
 		if err := w.Walk(); err != nil {
-			c.logger.ERROR.Println("walker: ", err)
+			c.logger.Errorln("walker: ", err)
 		}
 	}
 
@@ -740,8 +740,8 @@ func (c *commandeer) buildSites() (err error) {
 func (c *commandeer) handleBuildErr(err error, msg string) {
 	c.buildErr = err
 
-	c.logger.ERROR.Print(msg + ":\n\n")
-	c.logger.ERROR.Println(helpers.FirstUpper(err.Error()))
+	c.logger.Errorln(msg + ":\n")
+	c.logger.Errorln(helpers.FirstUpper(err.Error()))
 	if !c.h.quiet && c.h.verbose {
 		herrors.PrintStackTraceFromErr(err)
 	}
@@ -822,13 +822,13 @@ func (c *commandeer) fullRebuild(changeType string) {
 		if !c.paused {
 			_, err := c.copyStatic()
 			if err != nil {
-				c.logger.ERROR.Println(err)
+				c.logger.Errorln(err)
 				return
 			}
 
 			err = c.buildSites()
 			if err != nil {
-				c.logger.ERROR.Println(err)
+				c.logger.Errorln(err)
 			} else if !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload") {
 				livereload.ForceRefresh()
 			}
@@ -862,7 +862,7 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 	// Identifies changes to config (config.toml) files.
 	configSet := make(map[string]bool)
 
-	c.logger.FEEDBACK.Println("Watching for config changes in", strings.Join(c.configFiles, ", "))
+	c.logger.Println("Watching for config changes in", strings.Join(c.configFiles, ", "))
 	for _, configFile := range c.configFiles {
 		watcher.Add(configFile)
 		configSet[configFile] = true
@@ -879,7 +879,7 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 				}
 			case err := <-watcher.Errors:
 				if err != nil {
-					c.logger.ERROR.Println("Error while watching:", err)
+					c.logger.Errorln("Error while watching:", err)
 				}
 			}
 		}
@@ -895,9 +895,9 @@ func (c *commandeer) printChangeDetected(typ string) {
 	}
 	msg += " detected, rebuilding site."
 
-	c.logger.FEEDBACK.Println(msg)
+	c.logger.Println(msg)
 	const layout = "2006-01-02 15:04:05.000 -0700"
-	c.logger.FEEDBACK.Println(time.Now().Format(layout))
+	c.logger.Println(time.Now().Format(layout))
 }
 
 const (
@@ -979,7 +979,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
 		return
 	}
 
-	c.logger.INFO.Println("Received System Events:", evs)
+	c.logger.Infoln("Received System Events:", evs)
 
 	staticEvents := []fsnotify.Event{}
 	dynamicEvents := []fsnotify.Event{}
@@ -1059,7 +1059,7 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
 
 		walkAdder := func(path string, f hugofs.FileMetaInfo, err error) error {
 			if f.IsDir() {
-				c.logger.FEEDBACK.Println("adding created directory to watchlist", path)
+				c.logger.Println("adding created directory to watchlist", path)
 				if err := watcher.Add(path); err != nil {
 					return err
 				}
@@ -1091,15 +1091,15 @@ func (c *commandeer) handleEvents(watcher *watcher.Batcher,
 		c.printChangeDetected("Static files")
 
 		if c.Cfg.GetBool("forceSyncStatic") {
-			c.logger.FEEDBACK.Printf("Syncing all static files\n")
+			c.logger.Printf("Syncing all static files\n")
 			_, err := c.copyStatic()
 			if err != nil {
-				c.logger.ERROR.Println("Error copying static files to publish dir:", err)
+				c.logger.Errorln("Error copying static files to publish dir:", err)
 				return
 			}
 		} else {
 			if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil {
-				c.logger.ERROR.Println("Error syncing static files to publish dir:", err)
+				c.logger.Errorln("Error syncing static files to publish dir:", err)
 				return
 			}
 		}

commands/server.go

@@ -185,7 +185,7 @@ func (sc *serverCmd) server(cmd *cobra.Command, args []string) error {
 						// port set explicitly by user -- he/she probably meant it!
 						err = newSystemErrorF("Server startup failed: %s", err)
 					}
-					c.logger.FEEDBACK.Println("port", sc.serverPort, "already in use, attempting to use an available port")
+					c.logger.Println("port", sc.serverPort, "already in use, attempting to use an available port")
 					sp, err := helpers.FindAvailablePort()
 					if err != nil {
 						err = newSystemError("Unable to find alternative port to use:", err)
@@ -350,7 +350,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 					w.WriteHeader(500)
 					r, err := f.errorTemplate(err)
 					if err != nil {
-						f.c.logger.ERROR.Println(err)
+						f.c.logger.Errorln(err)
 					}
 
 					port = 1313
@@ -508,7 +508,7 @@ func (c *commandeer) serve(s *serverCmd) error {
 		go func() {
 			err = http.ListenAndServe(endpoint, mu)
 			if err != nil {
-				c.logger.ERROR.Printf("Error: %s\n", err.Error())
+				c.logger.Errorf("Error: %s\n", err.Error())
 				os.Exit(1)
 			}
 		}()

commands/static_syncer.go

@@ -107,10 +107,10 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 					logger.Println("Syncing", relPath, "to", publishDir)
 
 					if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-						c.logger.ERROR.Println(err)
+						c.logger.Errorln(err)
 					}
 				} else {
-					c.logger.ERROR.Println(err)
+					c.logger.Errorln(err)
 				}
 
 				continue
@@ -119,7 +119,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// For all other event operations Hugo will sync static.
 			logger.Println("Syncing", relPath, "to", publishDir)
 			if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-				c.logger.ERROR.Println(err)
+				c.logger.Errorln(err)
 			}
 		}
 

common/constants/constants.go

@@ -18,4 +18,8 @@ package constants
 const (
 	ErrIDAmbigousDisableKindTaxonomy = "error-disable-taxonomy"
 	ErrIDAmbigousOutputKindTaxonomy  = "error-output-taxonomy"
+
+	// IDs for remote errors in tpl/data.
+	ErrRemoteGetJSON = "error-remote-getjson"
+	ErrRemoteGetCSV  = "error-remote-getcsv"
 )

common/loggers/ignorableLogger.go

@@ -19,25 +19,31 @@ import (
 )
 
 // IgnorableLogger is a logger that ignores certain log statements.
-type IgnorableLogger struct {
-	logger     *Logger
+type IgnorableLogger interface {
+	Logger
+	Errorsf(statementID, format string, v ...interface{})
+}
+
+type ignorableLogger struct {
+	Logger
 	statements map[string]bool
 }
 
 // NewIgnorableLogger wraps the given logger and ignores the log statement IDs given.
-func NewIgnorableLogger(logger *Logger, statements ...string) IgnorableLogger {
+func NewIgnorableLogger(logger Logger, statements ...string) IgnorableLogger {
 	statementsSet := make(map[string]bool)
 	for _, s := range statements {
 		statementsSet[strings.ToLower(s)] = true
 
 	}
-	return IgnorableLogger{
-		logger:     logger,
+	return ignorableLogger{
+		Logger:     logger,
 		statements: statementsSet,
 	}
 }
 
-func (l IgnorableLogger) Errorf(statementID, format string, v ...interface{}) {
+// Errorsf logs statementID as an ERROR if not configured as ignoreable.
+func (l ignorableLogger) Errorsf(statementID, format string, v ...interface{}) {
 	if l.statements[statementID] {
 		// Ignore.
 		return
@@ -48,5 +54,5 @@ ignoreErrors = [%q]`, statementID)
 
 	format += ignoreMsg
 
-	l.logger.ERROR.Printf(format, v...)
+	l.Errorf(format, v...)
 }