Refactored all synchronizers to deal with some of the chronic issues.

2026-02-06 01:18:12 +01:00
parent d1425ca9ca
commit 071f1c9786
43 changed files with 2785 additions and 582 deletions
@@ -0,0 +1,78 @@
+using System;
+using System.Threading.Tasks;
+using Google;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+using Wino.Core.Integration.Processors;
+
+namespace Wino.Core.Synchronizers.Errors.Gmail;
+
+/// <summary>
+/// Handles Gmail history ID expiration errors.
+/// When history is no longer available, resets the account's history ID to force a full resync.
+/// </summary>
+public class GmailHistoryExpiredHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<GmailHistoryExpiredHandler>();
+    private readonly IGmailChangeProcessor _gmailChangeProcessor;
+
+    public GmailHistoryExpiredHandler(IGmailChangeProcessor gmailChangeProcessor)
+    {
+        _gmailChangeProcessor = gmailChangeProcessor;
+    }
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        // Gmail returns 404 when history ID is no longer valid
+        if (error.ErrorCode == 404)
+        {
+            var message = error.ErrorMessage?.ToLowerInvariant() ?? string.Empty;
+            return message.Contains("history") || message.Contains("notfound");
+        }
+
+        if (error.Exception is GoogleApiException googleEx)
+        {
+            if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
+            {
+                var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty;
+                return errorMessage.Contains("history") ||
+                       errorMessage.Contains("not found") ||
+                       errorMessage.Contains("starthistoryid");
+            }
+        }
+
+        return false;
+    }
+
+    public async Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "Gmail history ID expired for account {AccountName} ({AccountId}). Resetting to force full sync.",
+            error.Account?.Name, error.Account?.Id);
+
+        error.Severity = SynchronizerErrorSeverity.Recoverable;
+        error.Category = SynchronizerErrorCategory.ResourceNotFound;
+
+        // Reset the account's synchronization identifier (history ID)
+        if (error.Account != null)
+        {
+            try
+            {
+                await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(
+                    error.Account.Id, string.Empty).ConfigureAwait(false);
+
+                _logger.Information("Successfully reset Gmail history ID for account {AccountName}. Next sync will be full sync.",
+                    error.Account.Name);
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, "Failed to reset Gmail history ID for account {AccountName}",
+                    error.Account.Name);
+            }
+        }
+
+        return true;
+    }
+}
@@ -0,0 +1,57 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Google;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+
+namespace Wino.Core.Synchronizers.Errors.Gmail;
+
+/// <summary>
+/// Handles Gmail API quota exceeded errors (HTTP 403 with quota error).
+/// This is a more severe rate limit that indicates daily quota exhaustion.
+/// </summary>
+public class GmailQuotaExceededHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<GmailQuotaExceededHandler>();
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        if (error.Exception is GoogleApiException googleEx)
+        {
+            // Quota exceeded usually returns 403
+            if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.Forbidden)
+            {
+                var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty;
+                var errorReason = googleEx.Error?.Errors?.FirstOrDefault()?.Reason?.ToLowerInvariant() ?? string.Empty;
+
+                return errorMessage.Contains("quota") ||
+                       errorMessage.Contains("limit exceeded") ||
+                       errorReason.Contains("quota") ||
+                       errorReason.Contains("ratelimitexceeded") ||
+                       errorReason.Contains("userlimitexceeded");
+            }
+        }
+
+        return false;
+    }
+
+    public Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "Gmail API quota exceeded for account {AccountName} ({AccountId}). Sync will be paused.",
+            error.Account?.Name, error.Account?.Id);
+
+        // Quota exceeded is more severe - treat as fatal to prevent repeated failures
+        // The user will be notified and sync will resume after quota resets
+        error.Severity = SynchronizerErrorSeverity.Fatal;
+        error.Category = SynchronizerErrorCategory.RateLimit;
+
+        // Suggest a very long delay - quotas typically reset daily
+        error.RetryDelay = TimeSpan.FromHours(1);
+
+        return Task.FromResult(true);
+    }
+}
@@ -0,0 +1,47 @@
+using System;
+using System.Threading.Tasks;
+using Google;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+
+namespace Wino.Core.Synchronizers.Errors.Gmail;
+
+/// <summary>
+/// Handles Gmail API rate limiting errors (HTTP 429 Too Many Requests).
+/// Marks the error as transient with appropriate backoff delay.
+/// </summary>
+public class GmailRateLimitHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<GmailRateLimitHandler>();
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        if (error.ErrorCode == 429)
+            return true;
+
+        if (error.Exception is GoogleApiException googleEx)
+        {
+            return googleEx.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests ||
+                   (googleEx.Error?.Code == 429);
+        }
+
+        return false;
+    }
+
+    public Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "Gmail API rate limit hit for account {AccountName} ({AccountId}). Operation: {Operation}. Will retry with backoff.",
+            error.Account?.Name, error.Account?.Id, error.OperationType ?? "N/A");
+
+        error.Severity = SynchronizerErrorSeverity.Transient;
+        error.Category = SynchronizerErrorCategory.RateLimit;
+
+        // Gmail rate limits are usually per-user, suggest a longer delay
+        error.RetryDelay = TimeSpan.FromSeconds(10);
+
+        return Task.FromResult(true);
+    }
+}
@@ -0,0 +1,40 @@
+using System.Threading.Tasks;
+using MailKit.Security;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+
+namespace Wino.Core.Synchronizers.Errors.Imap;
+
+/// <summary>
+/// Handles IMAP authentication failures (AuthenticationException, SaslException).
+/// Marks the error as requiring re-authentication.
+/// </summary>
+public class ImapAuthenticationFailedHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<ImapAuthenticationFailedHandler>();
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        return error.Exception is AuthenticationException ||
+               error.Exception is SaslException ||
+               (error.ErrorMessage?.Contains("authentication", System.StringComparison.OrdinalIgnoreCase) ?? false);
+    }
+
+    public Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "IMAP authentication failed for account {AccountName} ({AccountId}). User needs to re-authenticate.",
+            error.Account?.Name, error.Account?.Id);
+
+        // Mark as requiring authentication - this will stop sync and notify user
+        error.Severity = SynchronizerErrorSeverity.AuthRequired;
+        error.Category = SynchronizerErrorCategory.Authentication;
+
+        // No point in retrying auth failures - credentials need to be updated
+        error.RetryDelay = null;
+
+        return Task.FromResult(true);
+    }
+}
@@ -0,0 +1,45 @@
+using System;
+using System.IO;
+using System.Net.Sockets;
+using System.Threading.Tasks;
+using MailKit;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+
+namespace Wino.Core.Synchronizers.Errors.Imap;
+
+/// <summary>
+/// Handles IMAP connection loss errors (IOException, SocketException, ServiceNotConnectedException).
+/// Marks the error as transient for retry with backoff.
+/// </summary>
+public class ImapConnectionLostHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<ImapConnectionLostHandler>();
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        return error.Exception is IOException ||
+               error.Exception is SocketException ||
+               error.Exception is ServiceNotConnectedException ||
+               error.Exception?.InnerException is IOException ||
+               error.Exception?.InnerException is SocketException;
+    }
+
+    public Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "IMAP connection lost for account {AccountName} ({AccountId}). Folder: {FolderName}. Operation: {Operation}. Will retry.",
+            error.Account?.Name, error.Account?.Id, error.FolderName ?? "N/A", error.OperationType ?? "N/A");
+
+        // Mark as transient - the RetryExecutor will handle the retry logic
+        error.Severity = SynchronizerErrorSeverity.Transient;
+        error.Category = SynchronizerErrorCategory.Network;
+
+        // Suggest a reasonable retry delay for connection issues
+        error.RetryDelay = TimeSpan.FromSeconds(2);
+
+        return Task.FromResult(true);
+    }
+}
@@ -0,0 +1,67 @@
+using System.Threading.Tasks;
+using MailKit;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+using Wino.Core.Integration.Processors;
+
+namespace Wino.Core.Synchronizers.Errors.Imap;
+
+/// <summary>
+/// Handles IMAP folder not found errors (FolderNotFoundException).
+/// Deletes the folder locally and allows sync to continue with other folders.
+/// </summary>
+public class ImapFolderNotFoundHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<ImapFolderNotFoundHandler>();
+    private readonly IImapChangeProcessor _imapChangeProcessor;
+
+    public ImapFolderNotFoundHandler(IImapChangeProcessor imapChangeProcessor)
+    {
+        _imapChangeProcessor = imapChangeProcessor;
+    }
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        return error.Exception is FolderNotFoundException ||
+               error.ErrorCode == 404 ||
+               (error.ErrorMessage?.Contains("folder not found", System.StringComparison.OrdinalIgnoreCase) ?? false) ||
+               (error.ErrorMessage?.Contains("mailbox not found", System.StringComparison.OrdinalIgnoreCase) ?? false);
+    }
+
+    public async Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        _logger.Warning(error.Exception,
+            "IMAP folder not found for account {AccountName} ({AccountId}). Folder: {FolderName} ({FolderId}). Removing locally.",
+            error.Account?.Name, error.Account?.Id, error.FolderName, error.FolderId);
+
+        // Mark as recoverable - sync can continue with other folders
+        error.Severity = SynchronizerErrorSeverity.Recoverable;
+        error.Category = SynchronizerErrorCategory.ResourceNotFound;
+
+        // Try to delete the folder locally if we have the folder ID
+        if (error.FolderId.HasValue && error.Account != null)
+        {
+            try
+            {
+                // Get the folder's remote ID from the exception if available
+                var remoteId = error.Exception is FolderNotFoundException fnf ? fnf.FolderName : null;
+
+                if (!string.IsNullOrEmpty(remoteId))
+                {
+                    await _imapChangeProcessor.DeleteFolderAsync(error.Account.Id, remoteId).ConfigureAwait(false);
+                    _logger.Information("Successfully deleted local folder {FolderName} after server deletion.",
+                        error.FolderName);
+                }
+            }
+            catch (System.Exception ex)
+            {
+                _logger.Warning(ex, "Failed to delete local folder {FolderName} ({FolderId})",
+                    error.FolderName, error.FolderId);
+            }
+        }
+
+        return true;
+    }
+}
@@ -0,0 +1,100 @@
+using System;
+using System.Threading.Tasks;
+using MailKit.Net.Imap;
+using Serilog;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+using Wino.Core.Domain.Models.Synchronization;
+
+namespace Wino.Core.Synchronizers.Errors.Imap;
+
+/// <summary>
+/// Handles generic IMAP protocol errors (ImapProtocolException, ImapCommandException).
+/// This is the catch-all handler for IMAP errors not handled by more specific handlers.
+/// </summary>
+public class ImapProtocolErrorHandler : ISynchronizerErrorHandler
+{
+    private readonly ILogger _logger = Log.ForContext<ImapProtocolErrorHandler>();
+
+    public bool CanHandle(SynchronizerErrorContext error)
+    {
+        // This is a catch-all for IMAP-related exceptions
+        return error.Exception is ImapProtocolException ||
+               error.Exception is ImapCommandException;
+    }
+
+    public Task<bool> HandleAsync(SynchronizerErrorContext error)
+    {
+        var severity = ClassifyProtocolError(error);
+        var category = SynchronizerErrorCategory.ProtocolError;
+
+        _logger.Warning(error.Exception,
+            "IMAP protocol error for account {AccountName} ({AccountId}). Folder: {FolderName}. Operation: {Operation}. Severity: {Severity}",
+            error.Account?.Name, error.Account?.Id, error.FolderName ?? "N/A", error.OperationType ?? "N/A", severity);
+
+        error.Severity = severity;
+        error.Category = category;
+
+        // For transient protocol errors, suggest a retry delay
+        if (severity == SynchronizerErrorSeverity.Transient)
+        {
+            error.RetryDelay = TimeSpan.FromSeconds(5);
+        }
+
+        return Task.FromResult(true);
+    }
+
+    /// <summary>
+    /// Classifies the protocol error to determine if it's transient, recoverable, or fatal.
+    /// </summary>
+    private static SynchronizerErrorSeverity ClassifyProtocolError(SynchronizerErrorContext error)
+    {
+        var message = error.ErrorMessage?.ToLowerInvariant() ?? string.Empty;
+        var exMessage = error.Exception?.Message?.ToLowerInvariant() ?? string.Empty;
+
+        // Check for rate limiting / throttling
+        if (message.Contains("too many") || message.Contains("rate limit") ||
+            message.Contains("throttl") || exMessage.Contains("too many"))
+        {
+            return SynchronizerErrorSeverity.Transient;
+        }
+
+        // Check for temporary server issues
+        if (message.Contains("try again") || message.Contains("temporary") ||
+            message.Contains("busy") || exMessage.Contains("try again"))
+        {
+            return SynchronizerErrorSeverity.Transient;
+        }
+
+        // Check for command-specific errors that are usually transient
+        if (error.Exception is ImapCommandException cmdEx)
+        {
+            // NO response usually means the operation failed but can be retried
+            if (cmdEx.Response == ImapCommandResponse.No)
+            {
+                // Unless it's a permanent failure indication
+                if (message.Contains("permanent") || message.Contains("invalid"))
+                {
+                    return SynchronizerErrorSeverity.Recoverable;
+                }
+                return SynchronizerErrorSeverity.Transient;
+            }
+
+            // BAD response usually indicates a protocol violation - don't retry
+            if (cmdEx.Response == ImapCommandResponse.Bad)
+            {
+                return SynchronizerErrorSeverity.Recoverable;
+            }
+        }
+
+        // Protocol exceptions that indicate connection issues
+        if (error.Exception is ImapProtocolException)
+        {
+            // Most protocol exceptions are connection-related and transient
+            return SynchronizerErrorSeverity.Transient;
+        }
+
+        // Default to recoverable for unknown protocol errors
+        return SynchronizerErrorSeverity.Recoverable;
+    }
+}
@@ -3,7 +3,7 @@ using Microsoft.Graph.Models.ODataErrors;
 using Microsoft.Kiota.Abstractions;
 using Serilog;
 using Wino.Core.Domain.Interfaces;
-using Wino.Core.Domain.Models.Errors;
+using Wino.Core.Domain.Models.Synchronization;
 using Wino.Core.Integration.Processors;

 namespace Wino.Core.Synchronizers.Errors.Outlook;
@@ -1,8 +1,8 @@
 using System.Threading.Tasks;
 using Microsoft.Kiota.Abstractions;
 using Wino.Core.Domain.Interfaces;
-using Wino.Core.Domain.Models.Errors;
 using Wino.Core.Domain.Models.Requests;
+using Wino.Core.Domain.Models.Synchronization;
 using Wino.Core.Requests.Bundles;

 namespace Wino.Core.Synchronizers.Errors.Outlook;