fix: implement retry logic for ComponentInformation to handle transient errors

cmd/cloud-init-server/main.go

@@ -52,6 +52,7 @@ var (
 	wireguardServer      string
 	wireguardOnly        bool
 	debug                bool
+	logFormat            string
 	wireGuardMiddleware  func(http.Handler) http.Handler
 	storageBackend       = "mem"           // Default to memstore
 	dbPath               = "cloud-init.db" // Default database path for quackstore
@@ -98,6 +99,7 @@ func setupFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&wireguardServer, "wireguard-server", getEnv("WIREGUARD_SERVER", ""), "WireGuard server IP address and network (e.g. 100.97.0.1/16)")
 	flags.BoolVar(&wireguardOnly, "wireguard-only", parseBool(getEnv("WIREGUARD_ONLY", "false")), "Only allow access to the cloud-init functions from the WireGuard subnet")
 	flags.BoolVar(&debug, "debug", parseBool(getEnv("DEBUG", "false")), "Enable debug logging")
+	flags.StringVar(&logFormat, "log-format", getEnv("LOG_FORMAT", "auto"), "Log format: json, console, or auto (auto detects TTY)")
 	flags.StringVar(&storageBackend, "storage-backend", getEnv("STORAGE_BACKEND", "mem"), "Storage backend to use (mem or quack)")
 	flags.StringVar(&dbPath, "db-path", getEnv("DB_PATH", "cloud-init.db"), "Path to the database file for quackstore backend")
 	flags.StringVar(&memPath, "mem-path", getEnv("MEM_PATH", ""), "Path to initial in-memory store configuration")
@@ -124,6 +126,7 @@ func bindViperToFlags() {
 	_ = viper.BindEnv("wireguard_server")
 	_ = viper.BindEnv("wireguard_only")
 	_ = viper.BindEnv("debug")
+	_ = viper.BindEnv("log_format")
 	_ = viper.BindEnv("storage_backend")
 	_ = viper.BindEnv("db_path")
 	_ = viper.BindEnv("mem_path")
@@ -159,7 +162,34 @@ func startServer() error {
 	// Setup logger
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
-	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+
+	// Configure log output based on format and TTY detection
+	switch strings.ToLower(logFormat) {
+	case "json":
+		log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
+	case "console":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:        os.Stderr,
+			NoColor:    false,
+			TimeFormat: time.RFC3339,
+		})
+	case "auto":
+		// Auto-detect: use JSON if not a TTY (container logs), console with no color otherwise
+		if isatty := func() bool {
+			fileInfo, _ := os.Stderr.Stat()
+			return (fileInfo.Mode() & os.ModeCharDevice) != 0
+		}(); isatty {
+			log.Logger = log.Output(zerolog.ConsoleWriter{
+				Out:        os.Stderr,
+				NoColor:    false,
+				TimeFormat: time.RFC3339,
+			})
+		} else {
+			log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
+		}
+	default:
+		log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger()
+	}
 
 	// Initialize storage backend
 	var err error

cmd/cloud-init-server/metadata_handlers.go

@@ -68,7 +68,7 @@ func MetaDataHandler(smd smdclient.SMDClientInterface, store cistore.Store) http
 			id = urlId
 		}
 		log.Debug().Msgf("Getting metadata for id: %s", id)
-		smdComponent, err := smd.ComponentInformation(id)
+		smdComponent, err := smd.ComponentInformationWithRetry(id, 3)
 		if err != nil {
 			if esr, ok := err.(smdclient.ErrSMDResponse); ok {
 				var msg string

internal/smdclient/FakeSMDClient.go

@@ -133,6 +133,10 @@ func (f *FakeSMDClient) ComponentInformation(id string) (base.Component, error)
 	return base.Component{}, errors.New("component not found in fake SMD client")
 }
 
+func (f *FakeSMDClient) ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error) {
+	return f.ComponentInformation(id)
+}
+
 func (f *FakeSMDClient) Summary() {
 	fmt.Printf("FakeSMDClient: %d components from %s to %s\n", len(f.components), f.rosetta_mapping[0].ComponentID, f.rosetta_mapping[len(f.rosetta_mapping)-1].ComponentID)
 	groupNames := make([]string, 0, len(f.groups))

internal/smdclient/SMDclient.go

@@ -27,6 +27,7 @@ type SMDClientInterface interface {
 	MACfromID(id string) (string, error)
 	GroupMembership(id string) ([]string, error)
 	ComponentInformation(id string) (base.Component, error)
+	ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error)
 	PopulateNodes()
 	ClusterName() string
 	AddWGIP(id string, wgip string) error
@@ -77,7 +78,7 @@ func NewSMDClient(clusterName, baseurl, jwtURL, accessToken, certPath string, in
 		certPool *x509.CertPool
 	)
 
-	c = &http.Client{Timeout: 2 * time.Second}
+	c = &http.Client{Timeout: 10 * time.Second}
 
 	// try and load the cert if path is provided first
 	if certPath != "" {
@@ -95,7 +96,7 @@ func NewSMDClient(clusterName, baseurl, jwtURL, accessToken, certPath string, in
 			RootCAs:            certPool,
 			InsecureSkipVerify: insecure,
 		},
-		DisableKeepAlives: true,
+		DisableKeepAlives: false,
 		Dial: (&net.Dialer{
 			Timeout:   120 * time.Second,
 			KeepAlive: 120 * time.Second,
@@ -408,6 +409,94 @@ func (s *SMDClient) ComponentInformation(id string) (base.Component, error) {
 	return node, nil
 }
 
+// ComponentInformationWithRetry wraps ComponentInformation with exponential backoff retry logic
+// for transient network errors and timeouts. This is critical during boot storms when SMD
+// may be temporarily overloaded.
+func (s *SMDClient) ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error) {
+	var lastErr error
+	var node base.Component
+
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		node, err := s.ComponentInformation(id)
+		if err == nil {
+			// Success - return immediately
+			return node, nil
+		}
+
+		lastErr = err
+
+		// Check if error is retryable (timeout or network error)
+		if !isRetryableError(err) {
+			// Non-retryable error (e.g., 404 Not Found, auth failure)
+			return node, err
+		}
+
+		// Don't sleep on the last attempt
+		if attempt < maxRetries-1 {
+			// Exponential backoff: 100ms, 200ms, 400ms, 800ms, etc.
+			backoff := time.Duration(100<<uint(attempt)) * time.Millisecond
+			log.Warn().
+				Str("component_id", id).
+				Int("attempt", attempt+1).
+				Int("max_retries", maxRetries).
+				Dur("backoff", backoff).
+				Err(err).
+				Msg("SMD request failed, retrying after backoff")
+			time.Sleep(backoff)
+		}
+	}
+
+	// All retries exhausted
+	log.Error().
+		Str("component_id", id).
+		Int("attempts", maxRetries).
+		Err(lastErr).
+		Msg("SMD request failed after all retry attempts")
+
+	return node, fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr)
+}
+
+// isRetryableError determines if an error should trigger a retry
+func isRetryableError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Timeout errors
+	if strings.Contains(errStr, "context deadline exceeded") ||
+		strings.Contains(errStr, "Client.Timeout exceeded") ||
+		strings.Contains(errStr, "timeout") ||
+		strings.Contains(errStr, "Timeout") {
+		return true
+	}
+
+	// Network errors
+	if strings.Contains(errStr, "connection refused") ||
+		strings.Contains(errStr, "connection reset") ||
+		strings.Contains(errStr, "network is unreachable") ||
+		strings.Contains(errStr, "no route to host") ||
+		strings.Contains(errStr, "broken pipe") ||
+		strings.Contains(errStr, "EOF") {
+		return true
+	}
+
+	// Temporary service unavailability (503)
+	if strings.Contains(errStr, "503") ||
+		strings.Contains(errStr, "Service Unavailable") {
+		return true
+	}
+
+	// Too Many Requests (429) - SMD may be rate limiting
+	if strings.Contains(errStr, "429") ||
+		strings.Contains(errStr, "Too Many Requests") {
+		return true
+	}
+
+	return false
+}
+
 func (s *SMDClient) AddWGIP(id string, wgip string) error {
 	s.nodesMutex.Lock()
 	defer s.nodesMutex.Unlock()