Refactored all synchronizers to deal with some of the chronic issues.

This commit is contained in:
Burak Kaan Köse
2026-02-06 01:18:12 +01:00
parent d1425ca9ca
commit 071f1c9786
43 changed files with 2785 additions and 582 deletions
@@ -106,6 +106,12 @@ public class MailAccount
[Ignore]
public MailAccountPreferences Preferences { get; set; }
/// <summary>
/// Last time folder structure was synchronized.
/// Used for optimization - skip folder sync if synced recently.
/// </summary>
public DateTime? LastFolderStructureSyncDate { get; set; }
/// <summary>
/// Gets whether the account can perform ProfileInformation sync type.
/// </summary>
@@ -4,5 +4,6 @@ public enum SynchronizationCompletedState
{
Success, // All succeeded.
Canceled, // Canceled by user or HTTP call.
Failed // Exception.
Failed, // Exception.
PartiallyCompleted // Some folders succeeded, some failed.
}
@@ -0,0 +1,47 @@
namespace Wino.Core.Domain.Enums;
/// <summary>
/// Categorizes synchronization errors by their root cause for targeted handling.
/// </summary>
public enum SynchronizerErrorCategory
{
/// <summary>
/// Network-related issues: connection timeouts, DNS failures, socket errors.
/// </summary>
Network,
/// <summary>
/// Authentication failures: invalid credentials, expired tokens, revoked access.
/// </summary>
Authentication,
/// <summary>
/// Rate limiting: too many requests (HTTP 429), quota exceeded.
/// </summary>
RateLimit,
/// <summary>
/// Resource not found: folder or message deleted externally (HTTP 404).
/// </summary>
ResourceNotFound,
/// <summary>
/// Server errors: internal server errors (HTTP 5xx), service unavailable.
/// </summary>
ServerError,
/// <summary>
/// Protocol errors: IMAP/SMTP command failures, malformed responses.
/// </summary>
ProtocolError,
/// <summary>
/// Validation errors: invalid data, constraint violations.
/// </summary>
Validation,
/// <summary>
/// Unknown or unclassified error.
/// </summary>
Unknown
}
@@ -0,0 +1,31 @@
namespace Wino.Core.Domain.Enums;
/// <summary>
/// Classifies the severity of synchronization errors to determine retry behavior.
/// </summary>
public enum SynchronizerErrorSeverity
{
/// <summary>
/// Transient error that should be retried with exponential backoff.
/// Examples: network timeout, temporary server unavailability, rate limiting.
/// </summary>
Transient,
/// <summary>
/// Error that can be recovered from by skipping the affected item/folder and continuing sync.
/// Examples: folder deleted externally, message not found, permission denied on single item.
/// </summary>
Recoverable,
/// <summary>
/// Fatal error that requires stopping synchronization and user intervention.
/// Examples: account disabled, server permanently unavailable, critical configuration error.
/// </summary>
Fatal,
/// <summary>
/// Authentication error that requires the user to re-authenticate.
/// Examples: token expired, password changed, OAuth refresh failed.
/// </summary>
AuthRequired
}
@@ -171,4 +171,19 @@ public interface IAccountService
/// <returns>Whether the notifications should be created after sync or not.</returns>
Task<bool> IsNotificationsEnabled(Guid accountId);
Task UpdateAccountCustomServerInformationAsync(CustomServerInformation customServerInformation);
/// <summary>
/// Updates the last folder structure sync date for the given account.
/// Used for optimization to skip folder sync if it was done recently.
/// </summary>
/// <param name="accountId">Account id.</param>
Task UpdateLastFolderStructureSyncDateAsync(Guid accountId);
/// <summary>
/// Checks if folder structure should be synced based on the configured interval.
/// Returns true if LastFolderStructureSyncDate is null or older than the interval.
/// </summary>
/// <param name="accountId">Account id.</param>
/// <param name="syncInterval">Minimum interval between folder syncs.</param>
Task<bool> ShouldSyncFolderStructureAsync(Guid accountId, TimeSpan syncInterval);
}
@@ -162,4 +162,12 @@ public interface IMailService
/// <param name="onlineArchiveMailIds">Retrieved MailCopy ids from search result.</param>
/// <returns>Result model that contains added and removed mail copy ids.</returns>
Task<GmailArchiveComparisonResult> GetGmailArchiveComparisonResultAsync(Guid archiveFolderId, List<string> onlineArchiveMailIds);
/// <summary>
/// Gets the most recent mail IDs for a folder.
/// Used for notification purposes after sync completes.
/// </summary>
/// <param name="folderId">Folder ID.</param>
/// <param name="count">Number of recent mails to return.</param>
Task<IEnumerable<string>> GetRecentMailIdsForFolderAsync(Guid folderId, int count);
}
@@ -0,0 +1,60 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Wino.Core.Domain.Models.Retry;
using Wino.Core.Domain.Models.Synchronization;
namespace Wino.Core.Domain.Interfaces;
/// <summary>
/// Executes operations with automatic retry and error handling support.
/// </summary>
public interface IRetryExecutor
{
/// <summary>
/// Executes an operation with automatic retry based on the specified policy.
/// </summary>
/// <typeparam name="T">The return type of the operation.</typeparam>
/// <param name="operation">The async operation to execute.</param>
/// <param name="policy">The retry policy to apply.</param>
/// <param name="errorContextFactory">Factory to create error context from exceptions.</param>
/// <param name="errorHandler">Optional error handler for custom error processing.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The result of the operation.</returns>
/// <exception cref="Exception">Thrown when all retries are exhausted or a fatal error occurs.</exception>
Task<T> ExecuteWithRetryAsync<T>(
Func<CancellationToken, Task<T>> operation,
RetryPolicy policy,
Func<Exception, SynchronizerErrorContext> errorContextFactory,
ISynchronizerErrorHandlerFactory errorHandler = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Executes an operation with automatic retry based on the specified policy (void return).
/// </summary>
/// <param name="operation">The async operation to execute.</param>
/// <param name="policy">The retry policy to apply.</param>
/// <param name="errorContextFactory">Factory to create error context from exceptions.</param>
/// <param name="errorHandler">Optional error handler for custom error processing.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <exception cref="Exception">Thrown when all retries are exhausted or a fatal error occurs.</exception>
Task ExecuteWithRetryAsync(
Func<CancellationToken, Task> operation,
RetryPolicy policy,
Func<Exception, SynchronizerErrorContext> errorContextFactory,
ISynchronizerErrorHandlerFactory errorHandler = null,
CancellationToken cancellationToken = default);
/// <summary>
/// Executes an operation with default retry policy.
/// </summary>
/// <typeparam name="T">The return type of the operation.</typeparam>
/// <param name="operation">The async operation to execute.</param>
/// <param name="errorContextFactory">Factory to create error context from exceptions.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>The result of the operation.</returns>
Task<T> ExecuteWithRetryAsync<T>(
Func<CancellationToken, Task<T>> operation,
Func<Exception, SynchronizerErrorContext> errorContextFactory,
CancellationToken cancellationToken = default);
}
@@ -0,0 +1,9 @@
using System.Threading.Tasks;
using Wino.Core.Domain.Models.Synchronization;
namespace Wino.Core.Domain.Interfaces;
public interface ISynchronizerErrorHandlerFactory
{
Task<bool> HandleErrorAsync(SynchronizerErrorContext error);
}
@@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
namespace Wino.Core.Domain.Models.Connectivity;
/// <summary>
/// Represents the health status of an IMAP connection pool.
/// </summary>
public class ConnectionPoolHealth
{
/// <summary>
/// Gets or sets the total number of connections in the pool (including IDLE).
/// </summary>
public int TotalConnections { get; set; }
/// <summary>
/// Gets or sets the number of connections available for use.
/// </summary>
public int AvailableConnections { get; set; }
/// <summary>
/// Gets or sets the number of connections currently in use.
/// </summary>
public int InUseConnections { get; set; }
/// <summary>
/// Gets or sets the number of connections that have failed and need reconnection.
/// </summary>
public int FailedConnections { get; set; }
/// <summary>
/// Gets or sets the number of connections currently reconnecting.
/// </summary>
public int ReconnectingConnections { get; set; }
/// <summary>
/// Gets or sets whether the dedicated IDLE connection is active and listening.
/// </summary>
public bool IdleConnectionActive { get; set; }
/// <summary>
/// Gets or sets the timestamp of the last health check.
/// </summary>
public DateTime LastHealthCheck { get; set; }
/// <summary>
/// Gets or sets recent issues encountered by the pool.
/// </summary>
public List<string> RecentIssues { get; set; } = [];
/// <summary>
/// Gets whether the pool is healthy (has minimum required connections).
/// </summary>
public bool IsHealthy => AvailableConnections >= 1 && FailedConnections == 0;
/// <summary>
/// Gets a summary of the pool health.
/// </summary>
public string Summary => $"Total: {TotalConnections}, Available: {AvailableConnections}, InUse: {InUseConnections}, Failed: {FailedConnections}, IDLE: {(IdleConnectionActive ? "Active" : "Inactive")}";
}
@@ -0,0 +1,105 @@
using System;
namespace Wino.Core.Domain.Models.Retry;
/// <summary>
/// Defines retry behavior for synchronization operations with exponential backoff.
/// </summary>
public class RetryPolicy
{
private static readonly Random _jitterRandom = new();
/// <summary>
/// Gets or sets the maximum number of retry attempts. Default is 3.
/// </summary>
public int MaxRetries { get; set; } = 3;
/// <summary>
/// Gets or sets the initial delay before the first retry. Default is 1 second.
/// </summary>
public TimeSpan InitialDelay { get; set; } = TimeSpan.FromSeconds(1);
/// <summary>
/// Gets or sets the multiplier for exponential backoff. Default is 2.0.
/// Each retry delay = previous delay * multiplier.
/// </summary>
public double BackoffMultiplier { get; set; } = 2.0;
/// <summary>
/// Gets or sets the maximum delay between retries. Default is 2 minutes.
/// </summary>
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromMinutes(2);
/// <summary>
/// Gets or sets whether to add random jitter to delays to prevent thundering herd.
/// Default is true.
/// </summary>
public bool UseJitter { get; set; } = true;
/// <summary>
/// Gets or sets the maximum jitter as a percentage of the delay (0.0 to 1.0).
/// Default is 0.25 (25%).
/// </summary>
public double JitterFactor { get; set; } = 0.25;
/// <summary>
/// Calculates the delay for the given retry attempt using exponential backoff.
/// </summary>
/// <param name="retryAttempt">The retry attempt number (1-based).</param>
/// <returns>The delay to wait before the retry.</returns>
public TimeSpan GetDelay(int retryAttempt)
{
if (retryAttempt <= 0)
return TimeSpan.Zero;
// Calculate base delay with exponential backoff
var baseDelayMs = InitialDelay.TotalMilliseconds * Math.Pow(BackoffMultiplier, retryAttempt - 1);
// Apply max delay cap
baseDelayMs = Math.Min(baseDelayMs, MaxDelay.TotalMilliseconds);
// Apply jitter if enabled
if (UseJitter)
{
var jitterRange = baseDelayMs * JitterFactor;
var jitter = (_jitterRandom.NextDouble() * 2 - 1) * jitterRange; // +/- jitter range
baseDelayMs = Math.Max(0, baseDelayMs + jitter);
}
return TimeSpan.FromMilliseconds(baseDelayMs);
}
/// <summary>
/// Creates a default retry policy suitable for most synchronization operations.
/// </summary>
public static RetryPolicy Default => new();
/// <summary>
/// Creates an aggressive retry policy with more attempts and shorter delays.
/// Suitable for transient network issues.
/// </summary>
public static RetryPolicy Aggressive => new()
{
MaxRetries = 5,
InitialDelay = TimeSpan.FromMilliseconds(500),
BackoffMultiplier = 1.5,
MaxDelay = TimeSpan.FromSeconds(30)
};
/// <summary>
/// Creates a conservative retry policy with longer delays.
/// Suitable for rate limiting scenarios.
/// </summary>
public static RetryPolicy RateLimited => new()
{
MaxRetries = 3,
InitialDelay = TimeSpan.FromSeconds(10),
BackoffMultiplier = 2.0,
MaxDelay = TimeSpan.FromMinutes(5)
};
/// <summary>
/// Creates a no-retry policy that doesn't retry on failure.
/// </summary>
public static RetryPolicy NoRetry => new() { MaxRetries = 0 };
}
@@ -0,0 +1,124 @@
using System;
using System.Collections.Generic;
using Wino.Core.Domain.Enums;
namespace Wino.Core.Domain.Models.Synchronization;
/// <summary>
/// Result of synchronizing a single folder.
/// Used for partial failure tracking when one folder fails but others succeed.
/// </summary>
public class FolderSyncResult
{
/// <summary>
/// Gets or sets the folder ID.
/// </summary>
public Guid FolderId { get; set; }
/// <summary>
/// Gets or sets the folder name for display purposes.
/// </summary>
public string FolderName { get; set; }
/// <summary>
/// Gets or sets whether the folder sync was successful.
/// </summary>
public bool Success { get; set; }
/// <summary>
/// Gets or sets the number of messages downloaded/synchronized.
/// </summary>
public int DownloadedCount { get; set; }
/// <summary>
/// Gets or sets the number of messages deleted locally (removed from server).
/// </summary>
public int DeletedCount { get; set; }
/// <summary>
/// Gets or sets the number of messages whose flags were updated.
/// </summary>
public int UpdatedCount { get; set; }
/// <summary>
/// Gets or sets the error message if sync failed.
/// </summary>
public string ErrorMessage { get; set; }
/// <summary>
/// Gets or sets the error severity if sync failed.
/// </summary>
public SynchronizerErrorSeverity? ErrorSeverity { get; set; }
/// <summary>
/// Gets or sets the error category if sync failed.
/// </summary>
public SynchronizerErrorCategory? ErrorCategory { get; set; }
/// <summary>
/// Gets or sets whether this folder was skipped (e.g., due to configuration).
/// </summary>
public bool WasSkipped { get; set; }
/// <summary>
/// Gets or sets the reason the folder was skipped.
/// </summary>
public string SkipReason { get; set; }
/// <summary>
/// Creates a successful folder sync result.
/// </summary>
public static FolderSyncResult Successful(Guid folderId, string folderName, int downloaded = 0, int deleted = 0, int updated = 0)
=> new()
{
FolderId = folderId,
FolderName = folderName,
Success = true,
DownloadedCount = downloaded,
DeletedCount = deleted,
UpdatedCount = updated
};
/// <summary>
/// Creates a failed folder sync result.
/// </summary>
public static FolderSyncResult Failed(Guid folderId, string folderName, string errorMessage,
SynchronizerErrorSeverity severity = SynchronizerErrorSeverity.Fatal,
SynchronizerErrorCategory category = SynchronizerErrorCategory.Unknown)
=> new()
{
FolderId = folderId,
FolderName = folderName,
Success = false,
ErrorMessage = errorMessage,
ErrorSeverity = severity,
ErrorCategory = category
};
/// <summary>
/// Creates a failed folder sync result from an error context.
/// </summary>
public static FolderSyncResult Failed(Guid folderId, string folderName, SynchronizerErrorContext errorContext)
=> new()
{
FolderId = folderId,
FolderName = folderName,
Success = false,
ErrorMessage = errorContext?.ErrorMessage ?? "Unknown error",
ErrorSeverity = errorContext?.Severity ?? SynchronizerErrorSeverity.Fatal,
ErrorCategory = errorContext?.Category ?? SynchronizerErrorCategory.Unknown
};
/// <summary>
/// Creates a skipped folder sync result.
/// </summary>
public static FolderSyncResult Skipped(Guid folderId, string folderName, string reason)
=> new()
{
FolderId = folderId,
FolderName = folderName,
Success = true, // Skipping is not a failure
WasSkipped = true,
SkipReason = reason
};
}
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json.Serialization;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Enums;
@@ -25,6 +26,55 @@ public class MailSynchronizationResult
public Exception Exception { get; set; }
/// <summary>
/// Gets or sets the results for each folder that was synchronized.
/// Enables partial failure tracking - some folders may succeed while others fail.
/// </summary>
public List<FolderSyncResult> FolderResults { get; set; } = [];
/// <summary>
/// Gets whether the synchronization had any partial failures.
/// True if at least one folder failed but others succeeded.
/// </summary>
[JsonIgnore]
public bool HasPartialFailures => FolderResults.Any(f => !f.Success) && FolderResults.Any(f => f.Success);
/// <summary>
/// Gets the number of folders that were successfully synchronized.
/// </summary>
[JsonIgnore]
public int SuccessfulFolderCount => FolderResults.Count(f => f.Success);
/// <summary>
/// Gets the number of folders that failed to synchronize.
/// </summary>
[JsonIgnore]
public int FailedFolderCount => FolderResults.Count(f => !f.Success);
/// <summary>
/// Gets the total number of messages downloaded across all folders.
/// </summary>
[JsonIgnore]
public int TotalDownloadedCount => FolderResults.Sum(f => f.DownloadedCount);
/// <summary>
/// Gets the total number of messages deleted across all folders.
/// </summary>
[JsonIgnore]
public int TotalDeletedCount => FolderResults.Sum(f => f.DeletedCount);
/// <summary>
/// Gets the total number of messages updated across all folders.
/// </summary>
[JsonIgnore]
public int TotalUpdatedCount => FolderResults.Sum(f => f.UpdatedCount);
/// <summary>
/// Gets the folders that failed to sync for error reporting.
/// </summary>
[JsonIgnore]
public IEnumerable<FolderSyncResult> FailedFolders => FolderResults.Where(f => !f.Success);
public static MailSynchronizationResult Empty => new() { CompletedState = SynchronizationCompletedState.Success };
// Mail synchronization
@@ -43,6 +93,28 @@ public class MailSynchronizationResult
CompletedState = SynchronizationCompletedState.Success
};
/// <summary>
/// Creates a completed result with folder-level results.
/// </summary>
public static MailSynchronizationResult CompletedWithFolderResults(
IEnumerable<MailCopy> downloadedMessages,
List<FolderSyncResult> folderResults)
{
var hasAnyFailure = folderResults.Any(f => !f.Success);
var hasAnySuccess = folderResults.Any(f => f.Success);
return new()
{
DownloadedMessages = downloadedMessages,
FolderResults = folderResults,
CompletedState = hasAnyFailure && !hasAnySuccess
? SynchronizationCompletedState.Failed
: hasAnyFailure
? SynchronizationCompletedState.PartiallyCompleted
: SynchronizationCompletedState.Success
};
}
public static MailSynchronizationResult Canceled => new() { CompletedState = SynchronizationCompletedState.Canceled };
public static MailSynchronizationResult Failed(Exception exception) => new()
{
@@ -0,0 +1,95 @@
using System;
using System.Collections.Generic;
using Wino.Core.Domain.Entities.Shared;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Interfaces;
namespace Wino.Core.Domain.Models.Synchronization;
/// <summary>
/// Contains context information about a synchronizer error
/// </summary>
public class SynchronizerErrorContext
{
/// <summary>
/// Account associated with the error
/// </summary>
public MailAccount Account { get; set; }
/// <summary>
/// Gets or sets the error code
/// </summary>
public int? ErrorCode { get; set; }
/// <summary>
/// Gets or sets the error message
/// </summary>
public string ErrorMessage { get; set; }
/// <summary>
/// Gets or sets the request bundle associated with the error
/// </summary>
public IRequestBundle RequestBundle { get; set; }
/// <summary>
/// Gets or sets additional data associated with the error
/// </summary>
public Dictionary<string, object> AdditionalData { get; set; } = new Dictionary<string, object>();
/// <summary>
/// Gets or sets the exception associated with the error
/// </summary>
public Exception Exception { get; set; }
/// <summary>
/// Gets or sets the severity of the error for retry decision making.
/// </summary>
public SynchronizerErrorSeverity Severity { get; set; } = SynchronizerErrorSeverity.Fatal;
/// <summary>
/// Gets or sets the category of the error for targeted handling.
/// </summary>
public SynchronizerErrorCategory Category { get; set; } = SynchronizerErrorCategory.Unknown;
/// <summary>
/// Gets or sets the current retry attempt count.
/// </summary>
public int RetryCount { get; set; }
/// <summary>
/// Gets or sets the maximum number of retries allowed.
/// </summary>
public int MaxRetries { get; set; } = 3;
/// <summary>
/// Gets or sets the suggested delay before retrying.
/// </summary>
public TimeSpan? RetryDelay { get; set; }
/// <summary>
/// Gets or sets the folder ID associated with the error for partial failure tracking.
/// </summary>
public Guid? FolderId { get; set; }
/// <summary>
/// Gets or sets the folder name for display purposes.
/// </summary>
public string FolderName { get; set; }
/// <summary>
/// Gets or sets the type of operation that failed.
/// Examples: "FolderSync", "MailSync", "RequestExecution", "Idle"
/// </summary>
public string OperationType { get; set; }
/// <summary>
/// Gets whether this error should be retried based on severity and retry count.
/// </summary>
public bool ShouldRetry => Severity == SynchronizerErrorSeverity.Transient && RetryCount < MaxRetries;
/// <summary>
/// Gets whether synchronization can continue despite this error.
/// </summary>
public bool CanContinueSync => Severity == SynchronizerErrorSeverity.Recoverable ||
(Severity == SynchronizerErrorSeverity.Transient && RetryCount >= MaxRetries);
}