Retry downloading in batches for Outlook

Wino.Core/Synchronizers/OutlookSynchronizer.cs

@@ -4,7 +4,6 @@ using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
-using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Text.Json;
@@ -21,8 +20,6 @@ using Microsoft.Graph.Models.ODataErrors;
 using Microsoft.Kiota.Abstractions;
 using Microsoft.Kiota.Abstractions.Authentication;
 using Microsoft.Kiota.Abstractions.Serialization;
-using Microsoft.Kiota.Http.HttpClientLibrary.Middleware;
-using Microsoft.Kiota.Http.HttpClientLibrary.Middleware.Options;
 using MimeKit;
 using MoreLinq.Extensions;
 using Serilog;
@@ -122,14 +119,6 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
 
         handlers.Add(GetMicrosoftImmutableIdHandler());
 
-        // Remove existing RetryHandler and add a new one with custom options.
-        var existingRetryHandler = handlers.FirstOrDefault(a => a is RetryHandler);
-        if (existingRetryHandler != null)
-            handlers.Remove(existingRetryHandler);
-
-        // Add custom one.
-        handlers.Add(GetRetryHandler());
-
         var httpClient = GraphClientFactory.Create(handlers);
         _graphClient = new GraphServiceClient(httpClient, new BaseBearerTokenAuthenticationProvider(tokenProvider));
 
@@ -141,30 +130,6 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
 
     private MicrosoftImmutableIdHandler GetMicrosoftImmutableIdHandler() => new();
 
-    private RetryHandler GetRetryHandler()
-    {
-        var options = new RetryHandlerOption()
-        {
-            ShouldRetry = (delay, attempt, httpResponse) =>
-            {
-                var statusCode = httpResponse.StatusCode;
-
-                return statusCode switch
-                {
-                    HttpStatusCode.ServiceUnavailable => true,
-                    HttpStatusCode.GatewayTimeout => true,
-                    (HttpStatusCode)429 => true,
-                    HttpStatusCode.Unauthorized => true,
-                    _ => false
-                };
-            },
-            Delay = 3,
-            MaxRetry = 3
-        };
-
-        return new RetryHandler(options);
-    }
-
     #endregion
 
 
@@ -482,7 +447,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
                 try
                 {
                     // Download all messages in this chunk concurrently
-                    var chunkDownloadedIds = await DownloadMessageMetadataBatchAsync(messageIdsToDownload, folder, cancellationToken).ConfigureAwait(false);
+                    var chunkDownloadedIds = await DownloadMessageMetadataBatchAsync(messageIdsToDownload, folder, true, cancellationToken).ConfigureAwait(false);
 
                     downloadedMessageIds.AddRange(chunkDownloadedIds);
 
@@ -533,7 +498,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
     /// Downloads metadata for a batch of messages using Graph SDK batch API (no MIME content).
     /// Processes up to 20 messages per batch request as per MaximumAllowedBatchRequestSize.
     /// </summary>
-    private async Task<List<string>> DownloadMessageMetadataBatchAsync(List<string> messageIds, MailItemFolder folder, CancellationToken cancellationToken)
+    private async Task<List<string>> DownloadMessageMetadataBatchAsync(List<string> messageIds, MailItemFolder folder, bool retryFailedOnce, CancellationToken cancellationToken)
     {
         if (messageIds == null || messageIds.Count == 0)
             return new List<string>();
@@ -561,6 +526,10 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
             return downloadedIds;
         }
 
+        // Store failed message ids to retry after.
+
+        List<string> failedMessageIds = new();
+
         // Process in batches of MaximumAllowedBatchRequestSize (20)
         var batches = messagesToDownload.Batch((int)MaximumAllowedBatchRequestSize);
 
@@ -623,6 +592,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
                         else
                         {
                             _logger.Warning("Failed to deserialize message {MailId} for folder {FolderName}", messageId, folder.FolderName);
+                            failedMessageIds.Add(messageId);
                         }
                     }
                     catch (ODataError odataError)
@@ -634,6 +604,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
                         }
                         else
                         {
+                            failedMessageIds.Add(messageId);
                             _logger.Error("OData error while downloading mail {MailId} for folder {FolderName}. Error: {Error}", messageId, folder.FolderName, odataError.Error?.Message);
                         }
                     }
@@ -645,28 +616,44 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
                             Account = Account,
                             ErrorCode = (int?)serviceException.ResponseStatusCode,
                             ErrorMessage = $"Service error during batch mail download: {serviceException.Message}",
-                            Exception = serviceException
+                            Exception = serviceException,
                         };
 
                         var handled = await _errorHandlingFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
 
                         if (!handled)
                         {
+                            failedMessageIds.Add(messageId);
                             _logger.Error(serviceException, "Unhandled service error while downloading mail {MailId} for folder {FolderName}. Error: {ErrorCode}", messageId, folder.FolderName, serviceException.ResponseStatusCode);
                         }
                     }
                     catch (Exception ex)
                     {
+                        failedMessageIds.Add(messageId);
                         _logger.Error(ex, "Error occurred while processing message {MailId} for folder {FolderName}", messageId, folder.FolderName);
                     }
                 }
             }
             catch (Exception ex)
             {
+                failedMessageIds.AddRange(batch);
+
                 _logger.Error(ex, "Error occurred during batch download for folder {FolderName}", folder.FolderName);
             }
         }
 
+        if (retryFailedOnce && failedMessageIds.Any())
+        {
+            // For a good cause wait a little bit.
+
+            await Task.Delay(3000);
+
+            // Do not retry here once again.
+            var failedDownloadedMessagIds = await DownloadMessageMetadataBatchAsync(failedMessageIds, folder, false, cancellationToken);
+
+            downloadedIds.Concat(failedDownloadedMessagIds);
+        }
+
         return downloadedIds;
     }
 
@@ -799,7 +786,7 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
             if (newMailIds.Any())
             {
                 _logger.Information("Downloading {Count} new mails from delta sync for folder {FolderName} (metadata only)", newMailIds.Count, folder.FolderName);
-                var deltaDownloadedIds = await DownloadMessageMetadataBatchAsync(newMailIds, folder, cancellationToken).ConfigureAwait(false);
+                var deltaDownloadedIds = await DownloadMessageMetadataBatchAsync(newMailIds, folder, true, cancellationToken).ConfigureAwait(false);
                 downloadedMessageIds.AddRange(deltaDownloadedIds);
             }