Log prometheus config reload error instead of stopping sidecar

Signed-off-by: Max Neverov <[email protected]>

pkg/reloader/reloader.go

@@ -218,9 +218,7 @@ func (r *Reloader) Watch(ctx context.Context) error {
 		}
 
 		if err := r.apply(ctx); err != nil {
-			// Critical error.
-			// TODO(bwplotka): There is no need to get process down in this case and decrease availability, handle the error in different way.
-			return err
+			level.Error(r.logger).Log("msg", "apply error", "err", err)
 		}
 	}
 }

pkg/reloader/reloader_test.go

@@ -97,6 +97,7 @@ config:
 	}()
 
 	reloadsSeen := 0
+	cnt := 0
 	for {
 		select {
 		case <-ctx.Done():
@@ -105,10 +106,6 @@ config:
 		}
 
 		rel := reloads.Load().(int)
-		if rel <= reloadsSeen {
-			// Nothing new.
-			continue
-		}
 
 		if reloadsSeen == 0 {
 			// Initial apply seen (without doing nothing).
@@ -138,8 +135,14 @@ config:
   b: 2
   c: 3
 `, string(f))
-			// All good, break
-			break
+
+			// change the mode so reloader can't read the file.
+			testutil.Ok(t, os.Chmod(input, os.ModeDir))
+			cnt += 1
+			// that was the second attempt. All good, break.
+			if cnt == 2 {
+				break
+			}
 		}
 		reloadsSeen = rel
 	}