diff --git a/cmd/cloud-init-server/main.go b/cmd/cloud-init-server/main.go index 298b198bd..962c15689 100644 --- a/cmd/cloud-init-server/main.go +++ b/cmd/cloud-init-server/main.go @@ -52,6 +52,7 @@ var ( wireguardServer string wireguardOnly bool debug bool + logFormat string wireGuardMiddleware func(http.Handler) http.Handler storageBackend = "mem" // Default to memstore dbPath = "cloud-init.db" // Default database path for quackstore @@ -98,6 +99,7 @@ func setupFlags(flags *pflag.FlagSet) { flags.StringVar(&wireguardServer, "wireguard-server", getEnv("WIREGUARD_SERVER", ""), "WireGuard server IP address and network (e.g. 100.97.0.1/16)") flags.BoolVar(&wireguardOnly, "wireguard-only", parseBool(getEnv("WIREGUARD_ONLY", "false")), "Only allow access to the cloud-init functions from the WireGuard subnet") flags.BoolVar(&debug, "debug", parseBool(getEnv("DEBUG", "false")), "Enable debug logging") + flags.StringVar(&logFormat, "log-format", getEnv("LOG_FORMAT", "auto"), "Log format: json, console, or auto (auto detects TTY)") flags.StringVar(&storageBackend, "storage-backend", getEnv("STORAGE_BACKEND", "mem"), "Storage backend to use (mem or quack)") flags.StringVar(&dbPath, "db-path", getEnv("DB_PATH", "cloud-init.db"), "Path to the database file for quackstore backend") flags.StringVar(&memPath, "mem-path", getEnv("MEM_PATH", ""), "Path to initial in-memory store configuration") @@ -124,6 +126,7 @@ func bindViperToFlags() { _ = viper.BindEnv("wireguard_server") _ = viper.BindEnv("wireguard_only") _ = viper.BindEnv("debug") + _ = viper.BindEnv("log_format") _ = viper.BindEnv("storage_backend") _ = viper.BindEnv("db_path") _ = viper.BindEnv("mem_path") @@ -159,7 +162,34 @@ func startServer() error { // Setup logger zerolog.TimeFieldFormat = zerolog.TimeFormatUnix zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack - log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) + + // Configure log output based on format and TTY detection + switch strings.ToLower(logFormat) { + case "json": + log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger() + case "console": + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + NoColor: false, + TimeFormat: time.RFC3339, + }) + case "auto": + // Auto-detect: use JSON if not a TTY (container logs), console with no color otherwise + if isatty := func() bool { + fileInfo, _ := os.Stderr.Stat() + return (fileInfo.Mode() & os.ModeCharDevice) != 0 + }(); isatty { + log.Logger = log.Output(zerolog.ConsoleWriter{ + Out: os.Stderr, + NoColor: false, + TimeFormat: time.RFC3339, + }) + } else { + log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger() + } + default: + log.Logger = zerolog.New(os.Stderr).With().Timestamp().Logger() + } // Initialize storage backend var err error diff --git a/cmd/cloud-init-server/metadata_handlers.go b/cmd/cloud-init-server/metadata_handlers.go index 559ae6985..85644fc78 100644 --- a/cmd/cloud-init-server/metadata_handlers.go +++ b/cmd/cloud-init-server/metadata_handlers.go @@ -68,7 +68,7 @@ func MetaDataHandler(smd smdclient.SMDClientInterface, store cistore.Store) http id = urlId } log.Debug().Msgf("Getting metadata for id: %s", id) - smdComponent, err := smd.ComponentInformation(id) + smdComponent, err := smd.ComponentInformationWithRetry(id, 3) if err != nil { if esr, ok := err.(smdclient.ErrSMDResponse); ok { var msg string diff --git a/internal/smdclient/FakeSMDClient.go b/internal/smdclient/FakeSMDClient.go index e3f6d6b88..8dadaacd8 100644 --- a/internal/smdclient/FakeSMDClient.go +++ b/internal/smdclient/FakeSMDClient.go @@ -133,6 +133,10 @@ func (f *FakeSMDClient) ComponentInformation(id string) (base.Component, error) return base.Component{}, errors.New("component not found in fake SMD client") } +func (f *FakeSMDClient) ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error) { + return f.ComponentInformation(id) +} + func (f *FakeSMDClient) Summary() { fmt.Printf("FakeSMDClient: %d components from %s to %s\n", len(f.components), f.rosetta_mapping[0].ComponentID, f.rosetta_mapping[len(f.rosetta_mapping)-1].ComponentID) groupNames := make([]string, 0, len(f.groups)) diff --git a/internal/smdclient/SMDclient.go b/internal/smdclient/SMDclient.go index eaa30dd05..6a3cb0001 100644 --- a/internal/smdclient/SMDclient.go +++ b/internal/smdclient/SMDclient.go @@ -27,6 +27,7 @@ type SMDClientInterface interface { MACfromID(id string) (string, error) GroupMembership(id string) ([]string, error) ComponentInformation(id string) (base.Component, error) + ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error) PopulateNodes() ClusterName() string AddWGIP(id string, wgip string) error @@ -77,7 +78,7 @@ func NewSMDClient(clusterName, baseurl, jwtURL, accessToken, certPath string, in certPool *x509.CertPool ) - c = &http.Client{Timeout: 2 * time.Second} + c = &http.Client{Timeout: 10 * time.Second} // try and load the cert if path is provided first if certPath != "" { @@ -95,7 +96,7 @@ func NewSMDClient(clusterName, baseurl, jwtURL, accessToken, certPath string, in RootCAs: certPool, InsecureSkipVerify: insecure, }, - DisableKeepAlives: true, + DisableKeepAlives: false, Dial: (&net.Dialer{ Timeout: 120 * time.Second, KeepAlive: 120 * time.Second, @@ -408,6 +409,94 @@ func (s *SMDClient) ComponentInformation(id string) (base.Component, error) { return node, nil } +// ComponentInformationWithRetry wraps ComponentInformation with exponential backoff retry logic +// for transient network errors and timeouts. This is critical during boot storms when SMD +// may be temporarily overloaded. +func (s *SMDClient) ComponentInformationWithRetry(id string, maxRetries int) (base.Component, error) { + var lastErr error + var node base.Component + + for attempt := 0; attempt < maxRetries; attempt++ { + node, err := s.ComponentInformation(id) + if err == nil { + // Success - return immediately + return node, nil + } + + lastErr = err + + // Check if error is retryable (timeout or network error) + if !isRetryableError(err) { + // Non-retryable error (e.g., 404 Not Found, auth failure) + return node, err + } + + // Don't sleep on the last attempt + if attempt < maxRetries-1 { + // Exponential backoff: 100ms, 200ms, 400ms, 800ms, etc. + backoff := time.Duration(100<