Refactored all synchronizers to deal with some of the chronic issues.

2026-02-06 01:18:12 +01:00
parent d1425ca9ca
commit 071f1c9786
43 changed files with 2785 additions and 582 deletions
@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+
+namespace Wino.Core.Domain.Models.Connectivity;
+
+/// <summary>
+/// Represents the health status of an IMAP connection pool.
+/// </summary>
+public class ConnectionPoolHealth
+{
+    /// <summary>
+    /// Gets or sets the total number of connections in the pool (including IDLE).
+    /// </summary>
+    public int TotalConnections { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of connections available for use.
+    /// </summary>
+    public int AvailableConnections { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of connections currently in use.
+    /// </summary>
+    public int InUseConnections { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of connections that have failed and need reconnection.
+    /// </summary>
+    public int FailedConnections { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of connections currently reconnecting.
+    /// </summary>
+    public int ReconnectingConnections { get; set; }
+
+    /// <summary>
+    /// Gets or sets whether the dedicated IDLE connection is active and listening.
+    /// </summary>
+    public bool IdleConnectionActive { get; set; }
+
+    /// <summary>
+    /// Gets or sets the timestamp of the last health check.
+    /// </summary>
+    public DateTime LastHealthCheck { get; set; }
+
+    /// <summary>
+    /// Gets or sets recent issues encountered by the pool.
+    /// </summary>
+    public List<string> RecentIssues { get; set; } = [];
+
+    /// <summary>
+    /// Gets whether the pool is healthy (has minimum required connections).
+    /// </summary>
+    public bool IsHealthy => AvailableConnections >= 1 && FailedConnections == 0;
+
+    /// <summary>
+    /// Gets a summary of the pool health.
+    /// </summary>
+    public string Summary => $"Total: {TotalConnections}, Available: {AvailableConnections}, InUse: {InUseConnections}, Failed: {FailedConnections}, IDLE: {(IdleConnectionActive ? "Active" : "Inactive")}";
+}
@@ -0,0 +1,105 @@
+using System;
+
+namespace Wino.Core.Domain.Models.Retry;
+
+/// <summary>
+/// Defines retry behavior for synchronization operations with exponential backoff.
+/// </summary>
+public class RetryPolicy
+{
+    private static readonly Random _jitterRandom = new();
+
+    /// <summary>
+    /// Gets or sets the maximum number of retry attempts. Default is 3.
+    /// </summary>
+    public int MaxRetries { get; set; } = 3;
+
+    /// <summary>
+    /// Gets or sets the initial delay before the first retry. Default is 1 second.
+    /// </summary>
+    public TimeSpan InitialDelay { get; set; } = TimeSpan.FromSeconds(1);
+
+    /// <summary>
+    /// Gets or sets the multiplier for exponential backoff. Default is 2.0.
+    /// Each retry delay = previous delay * multiplier.
+    /// </summary>
+    public double BackoffMultiplier { get; set; } = 2.0;
+
+    /// <summary>
+    /// Gets or sets the maximum delay between retries. Default is 2 minutes.
+    /// </summary>
+    public TimeSpan MaxDelay { get; set; } = TimeSpan.FromMinutes(2);
+
+    /// <summary>
+    /// Gets or sets whether to add random jitter to delays to prevent thundering herd.
+    /// Default is true.
+    /// </summary>
+    public bool UseJitter { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets the maximum jitter as a percentage of the delay (0.0 to 1.0).
+    /// Default is 0.25 (25%).
+    /// </summary>
+    public double JitterFactor { get; set; } = 0.25;
+
+    /// <summary>
+    /// Calculates the delay for the given retry attempt using exponential backoff.
+    /// </summary>
+    /// <param name="retryAttempt">The retry attempt number (1-based).</param>
+    /// <returns>The delay to wait before the retry.</returns>
+    public TimeSpan GetDelay(int retryAttempt)
+    {
+        if (retryAttempt <= 0)
+            return TimeSpan.Zero;
+
+        // Calculate base delay with exponential backoff
+        var baseDelayMs = InitialDelay.TotalMilliseconds * Math.Pow(BackoffMultiplier, retryAttempt - 1);
+
+        // Apply max delay cap
+        baseDelayMs = Math.Min(baseDelayMs, MaxDelay.TotalMilliseconds);
+
+        // Apply jitter if enabled
+        if (UseJitter)
+        {
+            var jitterRange = baseDelayMs * JitterFactor;
+            var jitter = (_jitterRandom.NextDouble() * 2 - 1) * jitterRange; // +/- jitter range
+            baseDelayMs = Math.Max(0, baseDelayMs + jitter);
+        }
+
+        return TimeSpan.FromMilliseconds(baseDelayMs);
+    }
+
+    /// <summary>
+    /// Creates a default retry policy suitable for most synchronization operations.
+    /// </summary>
+    public static RetryPolicy Default => new();
+
+    /// <summary>
+    /// Creates an aggressive retry policy with more attempts and shorter delays.
+    /// Suitable for transient network issues.
+    /// </summary>
+    public static RetryPolicy Aggressive => new()
+    {
+        MaxRetries = 5,
+        InitialDelay = TimeSpan.FromMilliseconds(500),
+        BackoffMultiplier = 1.5,
+        MaxDelay = TimeSpan.FromSeconds(30)
+    };
+
+    /// <summary>
+    /// Creates a conservative retry policy with longer delays.
+    /// Suitable for rate limiting scenarios.
+    /// </summary>
+    public static RetryPolicy RateLimited => new()
+    {
+        MaxRetries = 3,
+        InitialDelay = TimeSpan.FromSeconds(10),
+        BackoffMultiplier = 2.0,
+        MaxDelay = TimeSpan.FromMinutes(5)
+    };
+
+    /// <summary>
+    /// Creates a no-retry policy that doesn't retry on failure.
+    /// </summary>
+    public static RetryPolicy NoRetry => new() { MaxRetries = 0 };
+}
@@ -0,0 +1,124 @@
+using System;
+using System.Collections.Generic;
+using Wino.Core.Domain.Enums;
+
+namespace Wino.Core.Domain.Models.Synchronization;
+
+/// <summary>
+/// Result of synchronizing a single folder.
+/// Used for partial failure tracking when one folder fails but others succeed.
+/// </summary>
+public class FolderSyncResult
+{
+    /// <summary>
+    /// Gets or sets the folder ID.
+    /// </summary>
+    public Guid FolderId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the folder name for display purposes.
+    /// </summary>
+    public string FolderName { get; set; }
+
+    /// <summary>
+    /// Gets or sets whether the folder sync was successful.
+    /// </summary>
+    public bool Success { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of messages downloaded/synchronized.
+    /// </summary>
+    public int DownloadedCount { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of messages deleted locally (removed from server).
+    /// </summary>
+    public int DeletedCount { get; set; }
+
+    /// <summary>
+    /// Gets or sets the number of messages whose flags were updated.
+    /// </summary>
+    public int UpdatedCount { get; set; }
+
+    /// <summary>
+    /// Gets or sets the error message if sync failed.
+    /// </summary>
+    public string ErrorMessage { get; set; }
+
+    /// <summary>
+    /// Gets or sets the error severity if sync failed.
+    /// </summary>
+    public SynchronizerErrorSeverity? ErrorSeverity { get; set; }
+
+    /// <summary>
+    /// Gets or sets the error category if sync failed.
+    /// </summary>
+    public SynchronizerErrorCategory? ErrorCategory { get; set; }
+
+    /// <summary>
+    /// Gets or sets whether this folder was skipped (e.g., due to configuration).
+    /// </summary>
+    public bool WasSkipped { get; set; }
+
+    /// <summary>
+    /// Gets or sets the reason the folder was skipped.
+    /// </summary>
+    public string SkipReason { get; set; }
+
+    /// <summary>
+    /// Creates a successful folder sync result.
+    /// </summary>
+    public static FolderSyncResult Successful(Guid folderId, string folderName, int downloaded = 0, int deleted = 0, int updated = 0)
+        => new()
+        {
+            FolderId = folderId,
+            FolderName = folderName,
+            Success = true,
+            DownloadedCount = downloaded,
+            DeletedCount = deleted,
+            UpdatedCount = updated
+        };
+
+    /// <summary>
+    /// Creates a failed folder sync result.
+    /// </summary>
+    public static FolderSyncResult Failed(Guid folderId, string folderName, string errorMessage,
+        SynchronizerErrorSeverity severity = SynchronizerErrorSeverity.Fatal,
+        SynchronizerErrorCategory category = SynchronizerErrorCategory.Unknown)
+        => new()
+        {
+            FolderId = folderId,
+            FolderName = folderName,
+            Success = false,
+            ErrorMessage = errorMessage,
+            ErrorSeverity = severity,
+            ErrorCategory = category
+        };
+
+    /// <summary>
+    /// Creates a failed folder sync result from an error context.
+    /// </summary>
+    public static FolderSyncResult Failed(Guid folderId, string folderName, SynchronizerErrorContext errorContext)
+        => new()
+        {
+            FolderId = folderId,
+            FolderName = folderName,
+            Success = false,
+            ErrorMessage = errorContext?.ErrorMessage ?? "Unknown error",
+            ErrorSeverity = errorContext?.Severity ?? SynchronizerErrorSeverity.Fatal,
+            ErrorCategory = errorContext?.Category ?? SynchronizerErrorCategory.Unknown
+        };
+
+    /// <summary>
+    /// Creates a skipped folder sync result.
+    /// </summary>
+    public static FolderSyncResult Skipped(Guid folderId, string folderName, string reason)
+        => new()
+        {
+            FolderId = folderId,
+            FolderName = folderName,
+            Success = true, // Skipping is not a failure
+            WasSkipped = true,
+            SkipReason = reason
+        };
+}
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text.Json.Serialization;
 using Wino.Core.Domain.Entities.Mail;
 using Wino.Core.Domain.Enums;
@@ -25,6 +26,55 @@ public class MailSynchronizationResult

    public Exception Exception { get; set; }

+    /// <summary>
+    /// Gets or sets the results for each folder that was synchronized.
+    /// Enables partial failure tracking - some folders may succeed while others fail.
+    /// </summary>
+    public List<FolderSyncResult> FolderResults { get; set; } = [];
+
+    /// <summary>
+    /// Gets whether the synchronization had any partial failures.
+    /// True if at least one folder failed but others succeeded.
+    /// </summary>
+    [JsonIgnore]
+    public bool HasPartialFailures => FolderResults.Any(f => !f.Success) && FolderResults.Any(f => f.Success);
+
+    /// <summary>
+    /// Gets the number of folders that were successfully synchronized.
+    /// </summary>
+    [JsonIgnore]
+    public int SuccessfulFolderCount => FolderResults.Count(f => f.Success);
+
+    /// <summary>
+    /// Gets the number of folders that failed to synchronize.
+    /// </summary>
+    [JsonIgnore]
+    public int FailedFolderCount => FolderResults.Count(f => !f.Success);
+
+    /// <summary>
+    /// Gets the total number of messages downloaded across all folders.
+    /// </summary>
+    [JsonIgnore]
+    public int TotalDownloadedCount => FolderResults.Sum(f => f.DownloadedCount);
+
+    /// <summary>
+    /// Gets the total number of messages deleted across all folders.
+    /// </summary>
+    [JsonIgnore]
+    public int TotalDeletedCount => FolderResults.Sum(f => f.DeletedCount);
+
+    /// <summary>
+    /// Gets the total number of messages updated across all folders.
+    /// </summary>
+    [JsonIgnore]
+    public int TotalUpdatedCount => FolderResults.Sum(f => f.UpdatedCount);
+
+    /// <summary>
+    /// Gets the folders that failed to sync for error reporting.
+    /// </summary>
+    [JsonIgnore]
+    public IEnumerable<FolderSyncResult> FailedFolders => FolderResults.Where(f => !f.Success);
+
    public static MailSynchronizationResult Empty => new() { CompletedState = SynchronizationCompletedState.Success };

    // Mail synchronization
@@ -43,6 +93,28 @@ public class MailSynchronizationResult
            CompletedState = SynchronizationCompletedState.Success
        };

+    /// <summary>
+    /// Creates a completed result with folder-level results.
+    /// </summary>
+    public static MailSynchronizationResult CompletedWithFolderResults(
+        IEnumerable<MailCopy> downloadedMessages,
+        List<FolderSyncResult> folderResults)
+    {
+        var hasAnyFailure = folderResults.Any(f => !f.Success);
+        var hasAnySuccess = folderResults.Any(f => f.Success);
+
+        return new()
+        {
+            DownloadedMessages = downloadedMessages,
+            FolderResults = folderResults,
+            CompletedState = hasAnyFailure && !hasAnySuccess
+                ? SynchronizationCompletedState.Failed
+                : hasAnyFailure
+                    ? SynchronizationCompletedState.PartiallyCompleted
+                    : SynchronizationCompletedState.Success
+        };
+    }
+
    public static MailSynchronizationResult Canceled => new() { CompletedState = SynchronizationCompletedState.Canceled };
    public static MailSynchronizationResult Failed(Exception exception) => new()
    {
@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using Wino.Core.Domain.Entities.Shared;
+using Wino.Core.Domain.Enums;
+using Wino.Core.Domain.Interfaces;
+
+namespace Wino.Core.Domain.Models.Synchronization;
+
+/// <summary>
+/// Contains context information about a synchronizer error
+/// </summary>
+public class SynchronizerErrorContext
+{
+    /// <summary>
+    /// Account associated with the error
+    /// </summary>
+    public MailAccount Account { get; set; }
+
+    /// <summary>
+    /// Gets or sets the error code
+    /// </summary>
+    public int? ErrorCode { get; set; }
+
+    /// <summary>
+    /// Gets or sets the error message
+    /// </summary>
+    public string ErrorMessage { get; set; }
+
+    /// <summary>
+    /// Gets or sets the request bundle associated with the error
+    /// </summary>
+    public IRequestBundle RequestBundle { get; set; }
+
+    /// <summary>
+    /// Gets or sets additional data associated with the error
+    /// </summary>
+    public Dictionary<string, object> AdditionalData { get; set; } = new Dictionary<string, object>();
+
+    /// <summary>
+    /// Gets or sets the exception associated with the error
+    /// </summary>
+    public Exception Exception { get; set; }
+
+    /// <summary>
+    /// Gets or sets the severity of the error for retry decision making.
+    /// </summary>
+    public SynchronizerErrorSeverity Severity { get; set; } = SynchronizerErrorSeverity.Fatal;
+
+    /// <summary>
+    /// Gets or sets the category of the error for targeted handling.
+    /// </summary>
+    public SynchronizerErrorCategory Category { get; set; } = SynchronizerErrorCategory.Unknown;
+
+    /// <summary>
+    /// Gets or sets the current retry attempt count.
+    /// </summary>
+    public int RetryCount { get; set; }
+
+    /// <summary>
+    /// Gets or sets the maximum number of retries allowed.
+    /// </summary>
+    public int MaxRetries { get; set; } = 3;
+
+    /// <summary>
+    /// Gets or sets the suggested delay before retrying.
+    /// </summary>
+    public TimeSpan? RetryDelay { get; set; }
+
+    /// <summary>
+    /// Gets or sets the folder ID associated with the error for partial failure tracking.
+    /// </summary>
+    public Guid? FolderId { get; set; }
+
+    /// <summary>
+    /// Gets or sets the folder name for display purposes.
+    /// </summary>
+    public string FolderName { get; set; }
+
+    /// <summary>
+    /// Gets or sets the type of operation that failed.
+    /// Examples: "FolderSync", "MailSync", "RequestExecution", "Idle"
+    /// </summary>
+    public string OperationType { get; set; }
+
+    /// <summary>
+    /// Gets whether this error should be retried based on severity and retry count.
+    /// </summary>
+    public bool ShouldRetry => Severity == SynchronizerErrorSeverity.Transient && RetryCount < MaxRetries;
+
+    /// <summary>
+    /// Gets whether synchronization can continue despite this error.
+    /// </summary>
+    public bool CanContinueSync => Severity == SynchronizerErrorSeverity.Recoverable ||
+                                    (Severity == SynchronizerErrorSeverity.Transient && RetryCount >= MaxRetries);
+}