Add a retry logic to docker login operation (#3089)

While there's an existing retry mechanism for the `docker pull` command
[^1], it's missing for `docker login`.

Similar to the `docker pull` scenario, the container registry could
potentially be briefly unavailable or inaccessible, leading to failed
`docker login` attempt and subsequent workflow run failures.

Since it's container based workflow, there is not way to retry on
customer side. The runner should retry itself.

It also aligns with community feedback [^2].

[^1]: https://github.com/actions/runner/blob/8e0cd36cd8c74c3067ffe10189c1e42f7e753af2/src/Runner.Worker/ContainerOperationProvider.cs#L201
[^2]: https://github.com/orgs/community/discussions/73069

Co-authored-by: Thomas Boop <[email protected]>

Author: enescakir

src/Runner.Worker/ContainerOperationProvider.cs

@@ -466,17 +466,39 @@ private async Task<string> ContainerRegistryLogin(IExecutionContext executionCon
             {
                 throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
             }
-            var loginExitCode = await _dockerManager.DockerLogin(
-                executionContext,
-                configLocation,
-                container.RegistryServer,
-                container.RegistryAuthUsername,
-                container.RegistryAuthPassword);
 
-            if (loginExitCode != 0)
+            // Login docker with retry up to 3 times
+            int retryCount = 0;
+            int loginExitCode = 0;
+            while (retryCount < 3)
+            {
+                loginExitCode = await _dockerManager.DockerLogin(
+                    executionContext,
+                    configLocation,
+                    container.RegistryServer,
+                    container.RegistryAuthUsername,
+                    container.RegistryAuthPassword);
+                if (loginExitCode == 0)
+                {
+                    break;
+                }
+                else
+                {
+                    retryCount++;
+                    if (retryCount < 3)
+                    {
+                        var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
+                        executionContext.Warning($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
+                        await Task.Delay(backOff);
+                    }
+                }
+            }
+
+            if (retryCount == 3 && loginExitCode != 0)
             {
                 throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
             }
+
             return configLocation;
         }