diff --git a/internal/pkg/appconfig/types.go b/internal/pkg/appconfig/types.go index 05a6ff78..fd7fb137 100644 --- a/internal/pkg/appconfig/types.go +++ b/internal/pkg/appconfig/types.go @@ -130,6 +130,9 @@ type Config struct { EnableGPUBindUnbindWatch bool // Enable GPU bind/unbind event monitoring GPUBindUnbindPollInterval time.Duration // Poll interval for GPU bind/unbind events EnablePprof bool // Enable /debug/pprof/ HTTP endpoints + NVMLInitRetryAttempts int // Max attempts to initialize NVML before giving up + NVMLInitRetryBaseWait time.Duration // Initial backoff wait between NVML init retries + NVMLInitRetryMaxWait time.Duration // Cap on backoff wait between NVML init retries } // Clone returns a copy of Config with slices duplicated for reload snapshots. diff --git a/internal/pkg/nvmlprovider/provider.go b/internal/pkg/nvmlprovider/provider.go index 7633e0be..03355190 100644 --- a/internal/pkg/nvmlprovider/provider.go +++ b/internal/pkg/nvmlprovider/provider.go @@ -17,11 +17,14 @@ package nvmlprovider import ( + "context" "errors" "fmt" "log/slog" + "math/rand/v2" "strconv" "strings" + "time" "github.com/NVIDIA/go-nvml/pkg/nvml" ) @@ -32,16 +35,123 @@ type MIGDeviceInfo struct { ComputeInstanceID int } +// Default retry parameters for Initialize. The GPU driver installer on GKE +// (and similar node-bootstrap setups) can still be running when the exporter +// pod starts, so nvml.Init briefly returns ERROR_LIBRARY_NOT_FOUND. These +// defaults give the driver installer up to ~1 minute to finish before giving up. +const ( + DefaultNVMLInitRetryAttempts = 5 + DefaultNVMLInitRetryBaseWait = 2 * time.Second + DefaultNVMLInitRetryMaxWait = 20 * time.Second +) + +// maxJitterFraction is the maximum fraction of the backoff duration added as +// random jitter, so pods restarting together (e.g. after a node reboot) don't +// retry in lockstep against the same node. +const maxJitterFraction = 0.30 + var nvmlInterface NVML -// Initialize sets up the Singleton NVML interface. +// nvmlInitFunc is a package-level indirection over nvml.Init so tests can +// substitute a fake without requiring real GPU hardware. +var nvmlInitFunc = nvml.Init + +// nvmlInitError wraps the raw NVML return code from nvmlInitFunc so callers +// can distinguish transient failures (e.g. ERROR_LIBRARY_NOT_FOUND) from +// permanent ones without resorting to string matching. +type nvmlInitError struct { + ret nvml.Return +} + +func (e *nvmlInitError) Error() string { + return nvml.ErrorString(e.ret) +} + +// isLibraryNotFoundErr reports whether err represents nvml.ERROR_LIBRARY_NOT_FOUND, +// the transient error returned while the GPU driver installer hasn't finished yet. +func isLibraryNotFoundErr(err error) bool { + var initErr *nvmlInitError + if errors.As(err, &initErr) { + return initErr.ret == nvml.ERROR_LIBRARY_NOT_FOUND + } + return false +} + +// Initialize sets up the Singleton NVML interface, retrying on the transient +// ERROR_LIBRARY_NOT_FOUND error using sane default retry parameters. func Initialize() error { - var err error - nvmlInterface, err = newNVMLProvider() - if err != nil { - return err + return InitializeWithRetry(context.Background(), DefaultNVMLInitRetryAttempts, DefaultNVMLInitRetryBaseWait, DefaultNVMLInitRetryMaxWait) +} + +// InitializeWithRetry sets up the Singleton NVML interface, retrying up to +// attempts times with exponential backoff (base, 2x, 4x, ... capped at +// maxWait, plus jitter) when nvml.Init fails with ERROR_LIBRARY_NOT_FOUND. +// Any other error is returned immediately without retrying, since retrying a +// permanent failure only delays an unavoidable error. attempts < 1 is treated +// as 1, so at least one init attempt always happens. If ctx is cancelled +// while waiting between attempts, InitializeWithRetry returns ctx.Err() +// immediately instead of sleeping out the remaining backoff, so a shutdown +// signal during startup isn't ignored until retries are exhausted. +func InitializeWithRetry(ctx context.Context, attempts int, baseWait, maxWait time.Duration) error { + if attempts < 1 { + attempts = 1 } - return nil + + var lastErr error + for attempt := 0; attempt < attempts; attempt++ { + var err error + nvmlInterface, err = newNVMLProvider() + if err == nil { + return nil + } + lastErr = err + + if !isLibraryNotFoundErr(err) { + return fmt.Errorf("failed to initialize NVML library: %w", err) + } + + if attempt < attempts-1 { + wait := backoffDuration(attempt, baseWait, maxWait) + slog.Warn("NVML library not found yet (GPU driver may still be installing); retrying", + slog.Int("attempt", attempt+1), + slog.Int("maxAttempts", attempts), + slog.Duration("wait", wait)) + + timer := time.NewTimer(wait) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("NVML initialization cancelled after %d attempt(s): %w", attempt+1, ctx.Err()) + } + } + } + + return fmt.Errorf("failed to initialize NVML library after %d attempts, last error: %w", attempts, lastErr) +} + +// backoffDuration computes the exponential backoff wait for the given attempt +// (0-indexed): baseWait * 2^attempt, capped at maxWait, plus up to +// maxJitterFraction of additional random jitter on top of the cap. The +// doubling is computed via repeated, overflow-checked multiplication rather +// than a bit shift so an unbounded attempt count (attempts is a +// user-configurable CLI value) can never wrap into a bogus small positive +// duration that defeats the cap. +func backoffDuration(attempt int, baseWait, maxWait time.Duration) time.Duration { + wait := baseWait + for i := 0; i < attempt; i++ { + if wait >= maxWait || wait > maxWait/2 { + wait = maxWait + break + } + wait *= 2 + } + if wait > maxWait { + wait = maxWait + } + + jitter := time.Duration(rand.Float64() * maxJitterFraction * float64(wait)) //nolint:gosec // #nosec G404 -- jitter only needs to desynchronize retries, not be cryptographically secure + return wait + jitter } // reset clears the current NVML interface instance. @@ -77,9 +187,9 @@ func newNVMLProvider() (NVML, error) { } slog.Info("Attempting to initialize NVML library.") - ret := nvml.Init() + ret := nvmlInitFunc() if ret != nvml.SUCCESS { - err := errors.New(nvml.ErrorString(ret)) + err := &nvmlInitError{ret: ret} slog.Error(fmt.Sprintf("Cannot init NVML library; err: %v", err)) return nvmlProvider{initialized: false}, err } diff --git a/internal/pkg/nvmlprovider/provider_test.go b/internal/pkg/nvmlprovider/provider_test.go index 16b2daac..41850e6a 100644 --- a/internal/pkg/nvmlprovider/provider_test.go +++ b/internal/pkg/nvmlprovider/provider_test.go @@ -19,10 +19,22 @@ package nvmlprovider import ( "testing" + "github.com/NVIDIA/go-nvml/pkg/nvml" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) +// mockSuccessfulInit stubs nvmlInitFunc to succeed without a real NVML +// library, and returns a restore func the caller should defer. Only safe for +// tests that don't go on to call any other real nvml.* function: those +// still need the actual library loaded to resolve, and will crash the test +// binary with a symbol lookup error rather than return a Go error if it +// isn't. Use the Initialize()-then-t.Skip pattern elsewhere in this file for +// tests that do. +func mockSuccessfulInit() func() { + nvmlInitFunc = func() nvml.Return { return nvml.SUCCESS } + return func() { nvmlInitFunc = nvml.Init } +} + func TestGetMIGDeviceInfoByID_When_NVML_Not_Initialized(t *testing.T) { validMIGUUID := "MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/5" newNvmlProvider := nvmlProvider{} @@ -56,7 +68,14 @@ func TestGetAllMIGDevicesProcessMemory_When_NVML_Not_Initialized(t *testing.T) { } func TestGetMIGDeviceInfoByID_When_DriverVersion_Below_R470(t *testing.T) { - _ = Initialize() + // This exercises nvml.DeviceGetHandleByUUID beyond just Initialize, so a + // stubbed nvmlInitFunc isn't enough: the real NVML library still needs to + // be loaded for that symbol to resolve, or the test binary crashes + // outright instead of returning a Go error. Skip like its sibling tests + // when no real NVML is present. + if err := Initialize(); err != nil { + t.Skip("NVML not available, skipping test") + } assert.NotNil(t, Client(), "expected NVML Client to be not nil") assert.True(t, Client().(nvmlProvider).initialized, "expected Client to be initialized") defer Client().Cleanup() @@ -112,6 +131,8 @@ func TestGetMIGDeviceInfoByID_When_DriverVersion_Below_R470(t *testing.T) { } func Test_newNVMLProvider(t *testing.T) { + defer mockSuccessfulInit()() + tests := []struct { name string preRunFunc func() NVML @@ -195,9 +216,13 @@ func TestCleanup_WhenNotInitialized(t *testing.T) { // TestCleanup_WhenInitialized tests cleanup when NVML is initialized func TestCleanup_WhenInitialized(t *testing.T) { - // Initialize NVML + // Cleanup calls the real nvml.Shutdown, which needs the real library + // loaded to resolve, so this can't be satisfied by stubbing + // nvmlInitFunc alone. Skip like its sibling tests when unavailable. err := Initialize() - assert.NoError(t, err) + if err != nil { + t.Skip("NVML not available, skipping test") + } provider := Client() assert.NotNil(t, provider) @@ -239,9 +264,16 @@ func TestPreCheck(t *testing.T) { errorContains string }{ { + // This subtest's assertion below only cares that GetMIGDeviceInfoByID + // doesn't fail with "NVML not initialized" - it tolerates any other + // error - but reaching that call still needs Initialize to have + // actually succeeded, which needs the real NVML library. Skip like + // the other hardware-dependent tests in this file when unavailable. name: "Initialized provider", setupFunc: func(t *testing.T) { - require.NoError(t, Initialize()) + if err := Initialize(); err != nil { + t.Skip("NVML not available, skipping test") + } }, expectError: false, }, diff --git a/internal/pkg/nvmlprovider/retry_test.go b/internal/pkg/nvmlprovider/retry_test.go new file mode 100644 index 00000000..493b4f47 --- /dev/null +++ b/internal/pkg/nvmlprovider/retry_test.go @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nvmlprovider + +import ( + "context" + "testing" + "time" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestInitializeWithRetry_SucceedsAfterTransientFailures verifies that a couple +// of ERROR_LIBRARY_NOT_FOUND failures (e.g. the GPU driver installer is still +// running) are retried and initialization succeeds once the library becomes +// available. +func TestInitializeWithRetry_SucceedsAfterTransientFailures(t *testing.T) { + defer func() { nvmlInitFunc = nvml.Init }() + defer reset() + + callCount := 0 + nvmlInitFunc = func() nvml.Return { + callCount++ + if callCount <= 2 { + return nvml.ERROR_LIBRARY_NOT_FOUND + } + return nvml.SUCCESS + } + + err := InitializeWithRetry(context.Background(), 5, time.Millisecond, 5*time.Millisecond) + + require.NoError(t, err) + assert.Equal(t, 3, callCount, "expected exactly 3 calls: 2 failures + 1 success") + assert.True(t, Client().(nvmlProvider).initialized) +} + +// TestInitializeWithRetry_GivesUpAfterExhaustingAttempts verifies that a +// persistent transient error is retried up to the attempt limit and then +// returns a wrapped error naming the last failure. +func TestInitializeWithRetry_GivesUpAfterExhaustingAttempts(t *testing.T) { + defer func() { nvmlInitFunc = nvml.Init }() + defer reset() + + callCount := 0 + nvmlInitFunc = func() nvml.Return { + callCount++ + return nvml.ERROR_LIBRARY_NOT_FOUND + } + + err := InitializeWithRetry(context.Background(), 4, time.Millisecond, 5*time.Millisecond) + + require.Error(t, err) + assert.Equal(t, 4, callCount, "expected exactly 4 attempts") + assert.Contains(t, err.Error(), "4 attempts") + assert.Contains(t, err.Error(), nvml.ErrorString(nvml.ERROR_LIBRARY_NOT_FOUND)) +} + +// TestInitializeWithRetry_FailsFastOnNonTransientError verifies that an error +// other than ERROR_LIBRARY_NOT_FOUND is not retried, since retrying a +// permanent error only delays an unavoidable failure. +func TestInitializeWithRetry_FailsFastOnNonTransientError(t *testing.T) { + defer func() { nvmlInitFunc = nvml.Init }() + defer reset() + + callCount := 0 + nvmlInitFunc = func() nvml.Return { + callCount++ + return nvml.ERROR_INSUFFICIENT_POWER + } + + err := InitializeWithRetry(context.Background(), 5, time.Millisecond, 5*time.Millisecond) + + require.Error(t, err) + assert.Equal(t, 1, callCount, "non-transient error should fail after a single attempt") + assert.Contains(t, err.Error(), nvml.ErrorString(nvml.ERROR_INSUFFICIENT_POWER)) +} + +// TestInitializeWithRetry_ClampsNonPositiveAttemptsToOne verifies that a +// misconfigured attempts value (e.g. from a bad env var) still performs +// exactly one real init attempt instead of skipping initialization entirely. +func TestInitializeWithRetry_ClampsNonPositiveAttemptsToOne(t *testing.T) { + defer func() { nvmlInitFunc = nvml.Init }() + defer reset() + + for _, attempts := range []int{0, -1, -100} { + callCount := 0 + nvmlInitFunc = func() nvml.Return { + callCount++ + return nvml.ERROR_LIBRARY_NOT_FOUND + } + + err := InitializeWithRetry(context.Background(), attempts, time.Millisecond, time.Millisecond) + + require.Errorf(t, err, "attempts=%d", attempts) + assert.Equalf(t, 1, callCount, "attempts=%d should still perform exactly one init attempt", attempts) + assert.Containsf(t, err.Error(), "after 1 attempts", "attempts=%d", attempts) + } +} + +// TestInitializeWithRetry_ContextCancellationInterruptsBackoff verifies that +// cancelling ctx during the backoff wait returns promptly instead of sleeping +// out the full remaining backoff, so a shutdown signal during a slow NVML +// init retry isn't ignored until attempts are exhausted. +func TestInitializeWithRetry_ContextCancellationInterruptsBackoff(t *testing.T) { + defer func() { nvmlInitFunc = nvml.Init }() + defer reset() + + callCount := 0 + nvmlInitFunc = func() nvml.Return { + callCount++ + return nvml.ERROR_LIBRARY_NOT_FOUND + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + err := InitializeWithRetry(ctx, 5, 500*time.Millisecond, 500*time.Millisecond) + elapsed := time.Since(start) + + require.Error(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Less(t, elapsed, 200*time.Millisecond, + "cancellation should interrupt the backoff sleep instead of waiting it out") + assert.Less(t, callCount, 5, "should not have exhausted all attempts before the context was cancelled") +} + +// TestBackoffDuration_RespectsMaxWaitCap verifies the exponential backoff is +// capped at maxWait (plus jitter on top), rather than growing unbounded. +func TestBackoffDuration_RespectsMaxWaitCap(t *testing.T) { + baseWait := 2 * time.Second + maxWait := 20 * time.Second + + // A large attempt number would overflow the naive exponential without the cap. + d := backoffDuration(10, baseWait, maxWait) + + assert.GreaterOrEqual(t, d, maxWait, "backoff should never go below the capped wait") + assert.LessOrEqual(t, d, maxWait+time.Duration(float64(maxWait)*maxJitterFraction), + "backoff should not exceed maxWait plus the maximum jitter fraction") +} + +// TestBackoffDuration_JitterVaries proves jitter is actually applied by +// asserting repeated calls with identical inputs don't all return the same +// duration. +func TestBackoffDuration_JitterVaries(t *testing.T) { + baseWait := 2 * time.Second + maxWait := 20 * time.Second + + seen := make(map[time.Duration]bool) + for i := 0; i < 20; i++ { + d := backoffDuration(10, baseWait, maxWait) + assert.GreaterOrEqual(t, d, maxWait) + seen[d] = true + } + + assert.Greater(t, len(seen), 1, "expected varying durations across repeated calls due to jitter") +} + +// TestBackoffDuration_ExponentialGrowthBeforeCap verifies the backoff doubles +// on each attempt while still below maxWait, allowing for up to +// maxJitterFraction of jitter added on top of each scaled value. +func TestBackoffDuration_ExponentialGrowthBeforeCap(t *testing.T) { + baseWait := 100 * time.Millisecond + maxWait := 10 * time.Second + + for attempt, scaled := range map[int]time.Duration{ + 0: baseWait, + 1: baseWait * 2, + 2: baseWait * 4, + } { + d := backoffDuration(attempt, baseWait, maxWait) + maxWithJitter := scaled + time.Duration(float64(scaled)*maxJitterFraction) + + assert.GreaterOrEqualf(t, d, scaled, "attempt %d: backoff should be at least the scaled base", attempt) + assert.LessOrEqualf(t, d, maxWithJitter, "attempt %d: backoff should not exceed scaled base plus max jitter", attempt) + } +} + +// TestBackoffDuration_LargeAttemptDoesNotOverflowPastCap verifies that very +// large, user-configurable attempt counts (attempts is an unbounded CLI int) +// never wrap an overflowing multiplication into a bogus small duration that +// would defeat the maxWait cap and cause a retry storm. +func TestBackoffDuration_LargeAttemptDoesNotOverflowPastCap(t *testing.T) { + baseWait := 2 * time.Second + maxWait := 20 * time.Second + + for _, attempt := range []int{40, 63, 64, 1000, 1_000_000} { + d := backoffDuration(attempt, baseWait, maxWait) + + assert.GreaterOrEqualf(t, d, maxWait, "attempt %d: backoff must never fall below maxWait due to overflow", attempt) + assert.LessOrEqualf(t, d, maxWait+time.Duration(float64(maxWait)*maxJitterFraction), + "attempt %d: backoff must not exceed maxWait plus max jitter", attempt) + } +} diff --git a/pkg/cmd/app.go b/pkg/cmd/app.go index de0397df..11b2f5d6 100644 --- a/pkg/cmd/app.go +++ b/pkg/cmd/app.go @@ -111,12 +111,15 @@ const ( CLIEnableGPUBindUnbindWatch = "enable-gpu-bind-unbind-watch" CLIGPUBindUnbindPollInterval = "gpu-bind-unbind-poll-interval" CLIEnablePprof = "enable-pprof" + CLINVMLInitRetryAttempts = "nvml-init-retry-attempts" + CLINVMLInitRetryBaseWait = "nvml-init-retry-base-wait" + CLINVMLInitRetryMaxWait = "nvml-init-retry-max-wait" ) var ( validatePrerequisitesFunc = prerequisites.Validate initializeDCGMProviderFunc = dcgmprovider.Initialize - initializeNVMLProviderFunc = nvmlprovider.Initialize + initializeNVMLProviderFunc = nvmlprovider.InitializeWithRetry buildRegistryFunc = buildRegistry getCountersFunc = getCounters startWatchListManagerFunc = startDeviceWatchListManager @@ -391,6 +394,24 @@ func NewApp(buildVersion ...string) *cli.App { Usage: "Disable validation checks during startup. Can be useful for running in minimal environments or testing", EnvVars: []string{"DISABLE_STARTUP_VALIDATE"}, }, + &cli.IntFlag{ + Name: CLINVMLInitRetryAttempts, + Value: nvmlprovider.DefaultNVMLInitRetryAttempts, + Usage: "Max attempts to initialize NVML before giving up. Only retried on ERROR_LIBRARY_NOT_FOUND (e.g. the GPU driver installer hasn't finished yet), such as at pod startup on GKE.", + EnvVars: []string{"DCGM_EXPORTER_NVML_INIT_RETRY_ATTEMPTS"}, + }, + &cli.StringFlag{ + Name: CLINVMLInitRetryBaseWait, + Value: nvmlprovider.DefaultNVMLInitRetryBaseWait.String(), + Usage: "Initial backoff wait between NVML init retries, doubling each attempt up to nvml-init-retry-max-wait.", + EnvVars: []string{"DCGM_EXPORTER_NVML_INIT_RETRY_BASE_WAIT"}, + }, + &cli.StringFlag{ + Name: CLINVMLInitRetryMaxWait, + Value: nvmlprovider.DefaultNVMLInitRetryMaxWait.String(), + Usage: "Cap on backoff wait between NVML init retries.", + EnvVars: []string{"DCGM_EXPORTER_NVML_INIT_RETRY_MAX_WAIT"}, + }, &cli.BoolFlag{ Name: CLIEnableGPUBindUnbindWatch, Value: false, @@ -528,7 +549,7 @@ func runDCGMExporter(lifecycleCtx context.Context, c *cli.Context, reloadRequest // Initialize NVML Provider Instance only if Kubernetes mode is enabled // NVML is only needed for MIG device UUID parsing in Kubernetes environments if config.Kubernetes { - err = initializeNVMLProviderFunc() + err = initializeNVMLProviderFunc(lifecycleCtx, config.NVMLInitRetryAttempts, config.NVMLInitRetryBaseWait, config.NVMLInitRetryMaxWait) if err != nil && !config.DisableStartupValidate { return err } @@ -1111,7 +1132,13 @@ func (r *reloadCoordinator) doTopologyChange(ctx context.Context, reloadID uint6 nvmlprovider.Client().Cleanup() slog.InfoContext(ctx, "Reinitializing NVML", slog.Uint64("reload_id", reloadID)) - if err := nvmlprovider.Initialize(); err != nil { + // A single attempt (no retry) is deliberate here: unlike pod startup, a GPU + // unbind event routinely leaves no NVML library to find until the GPU is + // rebound, at which point a fresh topology-change event drives another call + // to this same path. This is the serial reload coordinator's event loop, so + // retrying with the full startup backoff would block /metrics recovery and + // any queued reload for the length of that backoff. + if err := nvmlprovider.InitializeWithRetry(ctx, 1, cfg.NVMLInitRetryBaseWait, cfg.NVMLInitRetryMaxWait); err != nil { slog.ErrorContext(ctx, "Failed to reinitialize NVML", slog.Uint64("reload_id", reloadID), slog.String("error", err.Error())) @@ -1482,6 +1509,9 @@ func defaultConfig() (*appconfig.Config, error) { EnableGPUBindUnbindWatch: false, GPUBindUnbindPollInterval: time.Second, EnablePprof: false, + NVMLInitRetryAttempts: nvmlprovider.DefaultNVMLInitRetryAttempts, + NVMLInitRetryBaseWait: nvmlprovider.DefaultNVMLInitRetryBaseWait, + NVMLInitRetryMaxWait: nvmlprovider.DefaultNVMLInitRetryMaxWait, }, nil } @@ -1616,6 +1646,15 @@ func applyExplicitConfigOverrides(c *cli.Context, config *appconfig.Config) erro if c.IsSet(CLIDisableStartupValidate) { config.DisableStartupValidate = c.Bool(CLIDisableStartupValidate) } + if c.IsSet(CLINVMLInitRetryAttempts) { + config.NVMLInitRetryAttempts = c.Int(CLINVMLInitRetryAttempts) + } + if c.IsSet(CLINVMLInitRetryBaseWait) { + config.NVMLInitRetryBaseWait = parseDuration(c.String(CLINVMLInitRetryBaseWait), nvmlprovider.DefaultNVMLInitRetryBaseWait) + } + if c.IsSet(CLINVMLInitRetryMaxWait) { + config.NVMLInitRetryMaxWait = parseDuration(c.String(CLINVMLInitRetryMaxWait), nvmlprovider.DefaultNVMLInitRetryMaxWait) + } if c.IsSet(CLIEnableGPUBindUnbindWatch) { config.EnableGPUBindUnbindWatch = c.Bool(CLIEnableGPUBindUnbindWatch) }