diff --git a/Wino.Core.Domain/Entities/Shared/MailAccount.cs b/Wino.Core.Domain/Entities/Shared/MailAccount.cs index 7370c268..c27008a4 100644 --- a/Wino.Core.Domain/Entities/Shared/MailAccount.cs +++ b/Wino.Core.Domain/Entities/Shared/MailAccount.cs @@ -106,6 +106,12 @@ public class MailAccount [Ignore] public MailAccountPreferences Preferences { get; set; } + /// + /// Last time folder structure was synchronized. + /// Used for optimization - skip folder sync if synced recently. + /// + public DateTime? LastFolderStructureSyncDate { get; set; } + /// /// Gets whether the account can perform ProfileInformation sync type. /// diff --git a/Wino.Core.Domain/Enums/SynchronizationCompletedState.cs b/Wino.Core.Domain/Enums/SynchronizationCompletedState.cs index 6a2edb6c..fc794169 100644 --- a/Wino.Core.Domain/Enums/SynchronizationCompletedState.cs +++ b/Wino.Core.Domain/Enums/SynchronizationCompletedState.cs @@ -4,5 +4,6 @@ public enum SynchronizationCompletedState { Success, // All succeeded. Canceled, // Canceled by user or HTTP call. - Failed // Exception. + Failed, // Exception. + PartiallyCompleted // Some folders succeeded, some failed. } diff --git a/Wino.Core.Domain/Enums/SynchronizerErrorCategory.cs b/Wino.Core.Domain/Enums/SynchronizerErrorCategory.cs new file mode 100644 index 00000000..4590273c --- /dev/null +++ b/Wino.Core.Domain/Enums/SynchronizerErrorCategory.cs @@ -0,0 +1,47 @@ +namespace Wino.Core.Domain.Enums; + +/// +/// Categorizes synchronization errors by their root cause for targeted handling. +/// +public enum SynchronizerErrorCategory +{ + /// + /// Network-related issues: connection timeouts, DNS failures, socket errors. + /// + Network, + + /// + /// Authentication failures: invalid credentials, expired tokens, revoked access. + /// + Authentication, + + /// + /// Rate limiting: too many requests (HTTP 429), quota exceeded. + /// + RateLimit, + + /// + /// Resource not found: folder or message deleted externally (HTTP 404). + /// + ResourceNotFound, + + /// + /// Server errors: internal server errors (HTTP 5xx), service unavailable. + /// + ServerError, + + /// + /// Protocol errors: IMAP/SMTP command failures, malformed responses. + /// + ProtocolError, + + /// + /// Validation errors: invalid data, constraint violations. + /// + Validation, + + /// + /// Unknown or unclassified error. + /// + Unknown +} diff --git a/Wino.Core.Domain/Enums/SynchronizerErrorSeverity.cs b/Wino.Core.Domain/Enums/SynchronizerErrorSeverity.cs new file mode 100644 index 00000000..49ce1d4c --- /dev/null +++ b/Wino.Core.Domain/Enums/SynchronizerErrorSeverity.cs @@ -0,0 +1,31 @@ +namespace Wino.Core.Domain.Enums; + +/// +/// Classifies the severity of synchronization errors to determine retry behavior. +/// +public enum SynchronizerErrorSeverity +{ + /// + /// Transient error that should be retried with exponential backoff. + /// Examples: network timeout, temporary server unavailability, rate limiting. + /// + Transient, + + /// + /// Error that can be recovered from by skipping the affected item/folder and continuing sync. + /// Examples: folder deleted externally, message not found, permission denied on single item. + /// + Recoverable, + + /// + /// Fatal error that requires stopping synchronization and user intervention. + /// Examples: account disabled, server permanently unavailable, critical configuration error. + /// + Fatal, + + /// + /// Authentication error that requires the user to re-authenticate. + /// Examples: token expired, password changed, OAuth refresh failed. + /// + AuthRequired +} diff --git a/Wino.Core.Domain/Interfaces/IAccountService.cs b/Wino.Core.Domain/Interfaces/IAccountService.cs index 6c590df7..cc31f0d9 100644 --- a/Wino.Core.Domain/Interfaces/IAccountService.cs +++ b/Wino.Core.Domain/Interfaces/IAccountService.cs @@ -171,4 +171,19 @@ public interface IAccountService /// Whether the notifications should be created after sync or not. Task IsNotificationsEnabled(Guid accountId); Task UpdateAccountCustomServerInformationAsync(CustomServerInformation customServerInformation); + + /// + /// Updates the last folder structure sync date for the given account. + /// Used for optimization to skip folder sync if it was done recently. + /// + /// Account id. + Task UpdateLastFolderStructureSyncDateAsync(Guid accountId); + + /// + /// Checks if folder structure should be synced based on the configured interval. + /// Returns true if LastFolderStructureSyncDate is null or older than the interval. + /// + /// Account id. + /// Minimum interval between folder syncs. + Task ShouldSyncFolderStructureAsync(Guid accountId, TimeSpan syncInterval); } diff --git a/Wino.Core.Domain/Interfaces/IMailService.cs b/Wino.Core.Domain/Interfaces/IMailService.cs index e6dfd7aa..317edbc0 100644 --- a/Wino.Core.Domain/Interfaces/IMailService.cs +++ b/Wino.Core.Domain/Interfaces/IMailService.cs @@ -162,4 +162,12 @@ public interface IMailService /// Retrieved MailCopy ids from search result. /// Result model that contains added and removed mail copy ids. Task GetGmailArchiveComparisonResultAsync(Guid archiveFolderId, List onlineArchiveMailIds); + + /// + /// Gets the most recent mail IDs for a folder. + /// Used for notification purposes after sync completes. + /// + /// Folder ID. + /// Number of recent mails to return. + Task> GetRecentMailIdsForFolderAsync(Guid folderId, int count); } diff --git a/Wino.Core.Domain/Interfaces/IRetryExecutor.cs b/Wino.Core.Domain/Interfaces/IRetryExecutor.cs new file mode 100644 index 00000000..e7d31c34 --- /dev/null +++ b/Wino.Core.Domain/Interfaces/IRetryExecutor.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Wino.Core.Domain.Models.Retry; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Domain.Interfaces; + +/// +/// Executes operations with automatic retry and error handling support. +/// +public interface IRetryExecutor +{ + /// + /// Executes an operation with automatic retry based on the specified policy. + /// + /// The return type of the operation. + /// The async operation to execute. + /// The retry policy to apply. + /// Factory to create error context from exceptions. + /// Optional error handler for custom error processing. + /// Cancellation token. + /// The result of the operation. + /// Thrown when all retries are exhausted or a fatal error occurs. + Task ExecuteWithRetryAsync( + Func> operation, + RetryPolicy policy, + Func errorContextFactory, + ISynchronizerErrorHandlerFactory errorHandler = null, + CancellationToken cancellationToken = default); + + /// + /// Executes an operation with automatic retry based on the specified policy (void return). + /// + /// The async operation to execute. + /// The retry policy to apply. + /// Factory to create error context from exceptions. + /// Optional error handler for custom error processing. + /// Cancellation token. + /// Thrown when all retries are exhausted or a fatal error occurs. + Task ExecuteWithRetryAsync( + Func operation, + RetryPolicy policy, + Func errorContextFactory, + ISynchronizerErrorHandlerFactory errorHandler = null, + CancellationToken cancellationToken = default); + + /// + /// Executes an operation with default retry policy. + /// + /// The return type of the operation. + /// The async operation to execute. + /// Factory to create error context from exceptions. + /// Cancellation token. + /// The result of the operation. + Task ExecuteWithRetryAsync( + Func> operation, + Func errorContextFactory, + CancellationToken cancellationToken = default); +} diff --git a/Wino.Core.Domain/Interfaces/ISynchronizerErrorHandlerFactory.cs b/Wino.Core.Domain/Interfaces/ISynchronizerErrorHandlerFactory.cs new file mode 100644 index 00000000..79f32f84 --- /dev/null +++ b/Wino.Core.Domain/Interfaces/ISynchronizerErrorHandlerFactory.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Domain.Interfaces; + +public interface ISynchronizerErrorHandlerFactory +{ + Task HandleErrorAsync(SynchronizerErrorContext error); +} diff --git a/Wino.Core.Domain/Models/Connectivity/ConnectionPoolHealth.cs b/Wino.Core.Domain/Models/Connectivity/ConnectionPoolHealth.cs new file mode 100644 index 00000000..a80424a8 --- /dev/null +++ b/Wino.Core.Domain/Models/Connectivity/ConnectionPoolHealth.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; + +namespace Wino.Core.Domain.Models.Connectivity; + +/// +/// Represents the health status of an IMAP connection pool. +/// +public class ConnectionPoolHealth +{ + /// + /// Gets or sets the total number of connections in the pool (including IDLE). + /// + public int TotalConnections { get; set; } + + /// + /// Gets or sets the number of connections available for use. + /// + public int AvailableConnections { get; set; } + + /// + /// Gets or sets the number of connections currently in use. + /// + public int InUseConnections { get; set; } + + /// + /// Gets or sets the number of connections that have failed and need reconnection. + /// + public int FailedConnections { get; set; } + + /// + /// Gets or sets the number of connections currently reconnecting. + /// + public int ReconnectingConnections { get; set; } + + /// + /// Gets or sets whether the dedicated IDLE connection is active and listening. + /// + public bool IdleConnectionActive { get; set; } + + /// + /// Gets or sets the timestamp of the last health check. + /// + public DateTime LastHealthCheck { get; set; } + + /// + /// Gets or sets recent issues encountered by the pool. + /// + public List RecentIssues { get; set; } = []; + + /// + /// Gets whether the pool is healthy (has minimum required connections). + /// + public bool IsHealthy => AvailableConnections >= 1 && FailedConnections == 0; + + /// + /// Gets a summary of the pool health. + /// + public string Summary => $"Total: {TotalConnections}, Available: {AvailableConnections}, InUse: {InUseConnections}, Failed: {FailedConnections}, IDLE: {(IdleConnectionActive ? "Active" : "Inactive")}"; +} diff --git a/Wino.Core.Domain/Models/Retry/RetryPolicy.cs b/Wino.Core.Domain/Models/Retry/RetryPolicy.cs new file mode 100644 index 00000000..e756a825 --- /dev/null +++ b/Wino.Core.Domain/Models/Retry/RetryPolicy.cs @@ -0,0 +1,105 @@ +using System; + +namespace Wino.Core.Domain.Models.Retry; + +/// +/// Defines retry behavior for synchronization operations with exponential backoff. +/// +public class RetryPolicy +{ + private static readonly Random _jitterRandom = new(); + + /// + /// Gets or sets the maximum number of retry attempts. Default is 3. + /// + public int MaxRetries { get; set; } = 3; + + /// + /// Gets or sets the initial delay before the first retry. Default is 1 second. + /// + public TimeSpan InitialDelay { get; set; } = TimeSpan.FromSeconds(1); + + /// + /// Gets or sets the multiplier for exponential backoff. Default is 2.0. + /// Each retry delay = previous delay * multiplier. + /// + public double BackoffMultiplier { get; set; } = 2.0; + + /// + /// Gets or sets the maximum delay between retries. Default is 2 minutes. + /// + public TimeSpan MaxDelay { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Gets or sets whether to add random jitter to delays to prevent thundering herd. + /// Default is true. + /// + public bool UseJitter { get; set; } = true; + + /// + /// Gets or sets the maximum jitter as a percentage of the delay (0.0 to 1.0). + /// Default is 0.25 (25%). + /// + public double JitterFactor { get; set; } = 0.25; + + /// + /// Calculates the delay for the given retry attempt using exponential backoff. + /// + /// The retry attempt number (1-based). + /// The delay to wait before the retry. + public TimeSpan GetDelay(int retryAttempt) + { + if (retryAttempt <= 0) + return TimeSpan.Zero; + + // Calculate base delay with exponential backoff + var baseDelayMs = InitialDelay.TotalMilliseconds * Math.Pow(BackoffMultiplier, retryAttempt - 1); + + // Apply max delay cap + baseDelayMs = Math.Min(baseDelayMs, MaxDelay.TotalMilliseconds); + + // Apply jitter if enabled + if (UseJitter) + { + var jitterRange = baseDelayMs * JitterFactor; + var jitter = (_jitterRandom.NextDouble() * 2 - 1) * jitterRange; // +/- jitter range + baseDelayMs = Math.Max(0, baseDelayMs + jitter); + } + + return TimeSpan.FromMilliseconds(baseDelayMs); + } + + /// + /// Creates a default retry policy suitable for most synchronization operations. + /// + public static RetryPolicy Default => new(); + + /// + /// Creates an aggressive retry policy with more attempts and shorter delays. + /// Suitable for transient network issues. + /// + public static RetryPolicy Aggressive => new() + { + MaxRetries = 5, + InitialDelay = TimeSpan.FromMilliseconds(500), + BackoffMultiplier = 1.5, + MaxDelay = TimeSpan.FromSeconds(30) + }; + + /// + /// Creates a conservative retry policy with longer delays. + /// Suitable for rate limiting scenarios. + /// + public static RetryPolicy RateLimited => new() + { + MaxRetries = 3, + InitialDelay = TimeSpan.FromSeconds(10), + BackoffMultiplier = 2.0, + MaxDelay = TimeSpan.FromMinutes(5) + }; + + /// + /// Creates a no-retry policy that doesn't retry on failure. + /// + public static RetryPolicy NoRetry => new() { MaxRetries = 0 }; +} diff --git a/Wino.Core.Domain/Models/Synchronization/FolderSyncResult.cs b/Wino.Core.Domain/Models/Synchronization/FolderSyncResult.cs new file mode 100644 index 00000000..2814fc2a --- /dev/null +++ b/Wino.Core.Domain/Models/Synchronization/FolderSyncResult.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using Wino.Core.Domain.Enums; + +namespace Wino.Core.Domain.Models.Synchronization; + +/// +/// Result of synchronizing a single folder. +/// Used for partial failure tracking when one folder fails but others succeed. +/// +public class FolderSyncResult +{ + /// + /// Gets or sets the folder ID. + /// + public Guid FolderId { get; set; } + + /// + /// Gets or sets the folder name for display purposes. + /// + public string FolderName { get; set; } + + /// + /// Gets or sets whether the folder sync was successful. + /// + public bool Success { get; set; } + + /// + /// Gets or sets the number of messages downloaded/synchronized. + /// + public int DownloadedCount { get; set; } + + /// + /// Gets or sets the number of messages deleted locally (removed from server). + /// + public int DeletedCount { get; set; } + + /// + /// Gets or sets the number of messages whose flags were updated. + /// + public int UpdatedCount { get; set; } + + /// + /// Gets or sets the error message if sync failed. + /// + public string ErrorMessage { get; set; } + + /// + /// Gets or sets the error severity if sync failed. + /// + public SynchronizerErrorSeverity? ErrorSeverity { get; set; } + + /// + /// Gets or sets the error category if sync failed. + /// + public SynchronizerErrorCategory? ErrorCategory { get; set; } + + /// + /// Gets or sets whether this folder was skipped (e.g., due to configuration). + /// + public bool WasSkipped { get; set; } + + /// + /// Gets or sets the reason the folder was skipped. + /// + public string SkipReason { get; set; } + + /// + /// Creates a successful folder sync result. + /// + public static FolderSyncResult Successful(Guid folderId, string folderName, int downloaded = 0, int deleted = 0, int updated = 0) + => new() + { + FolderId = folderId, + FolderName = folderName, + Success = true, + DownloadedCount = downloaded, + DeletedCount = deleted, + UpdatedCount = updated + }; + + /// + /// Creates a failed folder sync result. + /// + public static FolderSyncResult Failed(Guid folderId, string folderName, string errorMessage, + SynchronizerErrorSeverity severity = SynchronizerErrorSeverity.Fatal, + SynchronizerErrorCategory category = SynchronizerErrorCategory.Unknown) + => new() + { + FolderId = folderId, + FolderName = folderName, + Success = false, + ErrorMessage = errorMessage, + ErrorSeverity = severity, + ErrorCategory = category + }; + + /// + /// Creates a failed folder sync result from an error context. + /// + public static FolderSyncResult Failed(Guid folderId, string folderName, SynchronizerErrorContext errorContext) + => new() + { + FolderId = folderId, + FolderName = folderName, + Success = false, + ErrorMessage = errorContext?.ErrorMessage ?? "Unknown error", + ErrorSeverity = errorContext?.Severity ?? SynchronizerErrorSeverity.Fatal, + ErrorCategory = errorContext?.Category ?? SynchronizerErrorCategory.Unknown + }; + + /// + /// Creates a skipped folder sync result. + /// + public static FolderSyncResult Skipped(Guid folderId, string folderName, string reason) + => new() + { + FolderId = folderId, + FolderName = folderName, + Success = true, // Skipping is not a failure + WasSkipped = true, + SkipReason = reason + }; +} diff --git a/Wino.Core.Domain/Models/Synchronization/MailSynchronizationResult.cs b/Wino.Core.Domain/Models/Synchronization/MailSynchronizationResult.cs index 10dec4e4..c38fa6e3 100644 --- a/Wino.Core.Domain/Models/Synchronization/MailSynchronizationResult.cs +++ b/Wino.Core.Domain/Models/Synchronization/MailSynchronizationResult.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text.Json.Serialization; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Enums; @@ -25,6 +26,55 @@ public class MailSynchronizationResult public Exception Exception { get; set; } + /// + /// Gets or sets the results for each folder that was synchronized. + /// Enables partial failure tracking - some folders may succeed while others fail. + /// + public List FolderResults { get; set; } = []; + + /// + /// Gets whether the synchronization had any partial failures. + /// True if at least one folder failed but others succeeded. + /// + [JsonIgnore] + public bool HasPartialFailures => FolderResults.Any(f => !f.Success) && FolderResults.Any(f => f.Success); + + /// + /// Gets the number of folders that were successfully synchronized. + /// + [JsonIgnore] + public int SuccessfulFolderCount => FolderResults.Count(f => f.Success); + + /// + /// Gets the number of folders that failed to synchronize. + /// + [JsonIgnore] + public int FailedFolderCount => FolderResults.Count(f => !f.Success); + + /// + /// Gets the total number of messages downloaded across all folders. + /// + [JsonIgnore] + public int TotalDownloadedCount => FolderResults.Sum(f => f.DownloadedCount); + + /// + /// Gets the total number of messages deleted across all folders. + /// + [JsonIgnore] + public int TotalDeletedCount => FolderResults.Sum(f => f.DeletedCount); + + /// + /// Gets the total number of messages updated across all folders. + /// + [JsonIgnore] + public int TotalUpdatedCount => FolderResults.Sum(f => f.UpdatedCount); + + /// + /// Gets the folders that failed to sync for error reporting. + /// + [JsonIgnore] + public IEnumerable FailedFolders => FolderResults.Where(f => !f.Success); + public static MailSynchronizationResult Empty => new() { CompletedState = SynchronizationCompletedState.Success }; // Mail synchronization @@ -43,6 +93,28 @@ public class MailSynchronizationResult CompletedState = SynchronizationCompletedState.Success }; + /// + /// Creates a completed result with folder-level results. + /// + public static MailSynchronizationResult CompletedWithFolderResults( + IEnumerable downloadedMessages, + List folderResults) + { + var hasAnyFailure = folderResults.Any(f => !f.Success); + var hasAnySuccess = folderResults.Any(f => f.Success); + + return new() + { + DownloadedMessages = downloadedMessages, + FolderResults = folderResults, + CompletedState = hasAnyFailure && !hasAnySuccess + ? SynchronizationCompletedState.Failed + : hasAnyFailure + ? SynchronizationCompletedState.PartiallyCompleted + : SynchronizationCompletedState.Success + }; + } + public static MailSynchronizationResult Canceled => new() { CompletedState = SynchronizationCompletedState.Canceled }; public static MailSynchronizationResult Failed(Exception exception) => new() { diff --git a/Wino.Core.Domain/Models/Synchronization/SynchronizerErrorContext.cs b/Wino.Core.Domain/Models/Synchronization/SynchronizerErrorContext.cs new file mode 100644 index 00000000..9ba4c84c --- /dev/null +++ b/Wino.Core.Domain/Models/Synchronization/SynchronizerErrorContext.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using Wino.Core.Domain.Entities.Shared; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; + +namespace Wino.Core.Domain.Models.Synchronization; + +/// +/// Contains context information about a synchronizer error +/// +public class SynchronizerErrorContext +{ + /// + /// Account associated with the error + /// + public MailAccount Account { get; set; } + + /// + /// Gets or sets the error code + /// + public int? ErrorCode { get; set; } + + /// + /// Gets or sets the error message + /// + public string ErrorMessage { get; set; } + + /// + /// Gets or sets the request bundle associated with the error + /// + public IRequestBundle RequestBundle { get; set; } + + /// + /// Gets or sets additional data associated with the error + /// + public Dictionary AdditionalData { get; set; } = new Dictionary(); + + /// + /// Gets or sets the exception associated with the error + /// + public Exception Exception { get; set; } + + /// + /// Gets or sets the severity of the error for retry decision making. + /// + public SynchronizerErrorSeverity Severity { get; set; } = SynchronizerErrorSeverity.Fatal; + + /// + /// Gets or sets the category of the error for targeted handling. + /// + public SynchronizerErrorCategory Category { get; set; } = SynchronizerErrorCategory.Unknown; + + /// + /// Gets or sets the current retry attempt count. + /// + public int RetryCount { get; set; } + + /// + /// Gets or sets the maximum number of retries allowed. + /// + public int MaxRetries { get; set; } = 3; + + /// + /// Gets or sets the suggested delay before retrying. + /// + public TimeSpan? RetryDelay { get; set; } + + /// + /// Gets or sets the folder ID associated with the error for partial failure tracking. + /// + public Guid? FolderId { get; set; } + + /// + /// Gets or sets the folder name for display purposes. + /// + public string FolderName { get; set; } + + /// + /// Gets or sets the type of operation that failed. + /// Examples: "FolderSync", "MailSync", "RequestExecution", "Idle" + /// + public string OperationType { get; set; } + + /// + /// Gets whether this error should be retried based on severity and retry count. + /// + public bool ShouldRetry => Severity == SynchronizerErrorSeverity.Transient && RetryCount < MaxRetries; + + /// + /// Gets whether synchronization can continue despite this error. + /// + public bool CanContinueSync => Severity == SynchronizerErrorSeverity.Recoverable || + (Severity == SynchronizerErrorSeverity.Transient && RetryCount >= MaxRetries); +} diff --git a/Wino.Core/CoreContainerSetup.cs b/Wino.Core/CoreContainerSetup.cs index 3189706d..1f9d2ce0 100644 --- a/Wino.Core/CoreContainerSetup.cs +++ b/Wino.Core/CoreContainerSetup.cs @@ -4,6 +4,8 @@ using Wino.Authentication; using Wino.Core.Domain.Interfaces; using Wino.Core.Integration.Processors; using Wino.Core.Services; +using Wino.Core.Synchronizers.Errors.Gmail; +using Wino.Core.Synchronizers.Errors.Imap; using Wino.Core.Synchronizers.Errors.Outlook; using Wino.Core.Synchronizers.ImapSync; @@ -37,12 +39,29 @@ public static class CoreContainerSetup services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); - // Register error factory handlers + // Register Outlook error handlers services.AddTransient(); services.AddTransient(); + // Register Gmail error handlers + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + // Register IMAP error handlers + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + // Register error handler factories services.AddTransient(); services.AddTransient(); + services.AddTransient(); + + // Register retry executor + services.AddTransient(); } } diff --git a/Wino.Core/Domain/Interfaces/ISynchronizerErrorHandler.cs b/Wino.Core/Domain/Interfaces/ISynchronizerErrorHandler.cs index 6e5dcad2..f3a8e995 100644 --- a/Wino.Core/Domain/Interfaces/ISynchronizerErrorHandler.cs +++ b/Wino.Core/Domain/Interfaces/ISynchronizerErrorHandler.cs @@ -1,5 +1,5 @@ using System.Threading.Tasks; -using Wino.Core.Domain.Models.Errors; +using Wino.Core.Domain.Models.Synchronization; namespace Wino.Core.Domain.Interfaces; @@ -23,10 +23,6 @@ public interface ISynchronizerErrorHandler Task HandleAsync(SynchronizerErrorContext error); } -public interface ISynchronizerErrorHandlerFactory -{ - Task HandleErrorAsync(SynchronizerErrorContext error); -} - public interface IOutlookSynchronizerErrorHandlerFactory : ISynchronizerErrorHandlerFactory; public interface IGmailSynchronizerErrorHandlerFactory : ISynchronizerErrorHandlerFactory; +public interface IImapSynchronizerErrorHandlerFactory : ISynchronizerErrorHandlerFactory; diff --git a/Wino.Core/Domain/Models/Errors/SynchronizerErrorContext.cs b/Wino.Core/Domain/Models/Errors/SynchronizerErrorContext.cs deleted file mode 100644 index 4f868ea3..00000000 --- a/Wino.Core/Domain/Models/Errors/SynchronizerErrorContext.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections.Generic; -using Wino.Core.Domain.Entities.Shared; -using Wino.Core.Domain.Interfaces; - -namespace Wino.Core.Domain.Models.Errors; - -/// -/// Contains context information about a synchronizer error -/// -public class SynchronizerErrorContext -{ - /// - /// Account associated with the error - /// - public MailAccount Account { get; set; } - - /// - /// Gets or sets the error code - /// - public int? ErrorCode { get; set; } - - /// - /// Gets or sets the error message - /// - public string ErrorMessage { get; set; } - - /// - /// Gets or sets the request bundle associated with the error - /// - public IRequestBundle RequestBundle { get; set; } - - /// - /// Gets or sets additional data associated with the error - /// - public Dictionary AdditionalData { get; set; } = new Dictionary(); - - /// - /// Gets or sets the exception associated with the error - /// - public Exception Exception { get; set; } -} diff --git a/Wino.Core/Integration/ImapClientPool.cs b/Wino.Core/Integration/ImapClientPool.cs index 683c33aa..5114f2e2 100644 --- a/Wino.Core/Integration/ImapClientPool.cs +++ b/Wino.Core/Integration/ImapClientPool.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.IO; using System.Net; @@ -6,13 +6,12 @@ using System.Net.Security; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; -using System.Timers; using MailKit.Net.Imap; using MailKit.Net.Proxy; using MailKit.Security; using MimeKit.Cryptography; -using MoreLinq; using Serilog; using Wino.Core.Domain.Entities.Shared; using Wino.Core.Domain.Enums; @@ -20,19 +19,32 @@ using Wino.Core.Domain.Exceptions; using Wino.Core.Domain.Models.Connectivity; namespace Wino.Core.Integration; + /// -/// Provides a pooling mechanism for ImapClient. -/// Makes sure that we don't have too many connections to the server. -/// Rents a connected & authenticated client from the pool all the time. +/// Connection state for tracking individual client health. +/// +public enum ImapClientState +{ + Available, + InUse, + Idle, + Reconnecting, + Failed, + Disposed +} + +/// +/// Provides an enhanced pooling mechanism for ImapClient with Channel-based async rental. +/// Maintains minimum active connections and a dedicated IDLE client. /// -/// Connection/Authentication info to be used to configure ImapClient. public class ImapClientPool : IDisposable { - // Hardcoded implementation details for ID extension if the server supports. - // Some providers like Chinese 126 require Id to be sent before authentication. - // We don't expose any customer data here. Therefore it's safe for now. - // Later on maybe we can make it configurable and leave it to the user with passing - // real implementation details. + private const int MinActiveConnections = 3; + private const int IdleConnectionReserved = 1; + private const int KeepAliveIntervalMs = 4 * 60 * 1000; // 4 minutes + private const int ConnectionMonitorIntervalMs = 30 * 1000; // 30 seconds + private const int MaintenanceIntervalMs = 60 * 1000; // 1 minute + private readonly ImapImplementation _implementation = new() { Version = "1.8.0", @@ -42,180 +54,520 @@ public class ImapClientPool : IDisposable Name = "Wino Mail User", }; + private readonly ILogger _logger = Log.ForContext(); + private readonly CustomServerInformation _customServerInformation; + private readonly Stream _protocolLogStream; + private readonly ConcurrentDictionary _clientStates = new(); + private readonly Channel _availableClients; + private readonly CancellationTokenSource _maintenanceCts = new(); + private readonly object _idleClientLock = new(); + + private WinoImapClient _dedicatedIdleClient; + private bool _disposedValue; + private bool _initialized; + private Task _maintenanceTask; + public bool ThrowOnSSLHandshakeCallback { get; set; } public ImapClientPoolOptions ImapClientPoolOptions { get; } - private bool _disposedValue; - private readonly int MinimumPoolSize = 5; - private readonly ConcurrentStack _clients = new(); - private readonly SemaphoreSlim _semaphore; - private readonly CustomServerInformation _customServerInformation; - private readonly Stream _protocolLogStream; - private readonly ILogger _logger = Log.ForContext(); - private readonly System.Timers.Timer _keepAliveTimer; - private readonly System.Timers.Timer _connectionMonitorTimer; - - private const int KeepAliveInterval = 4 * 60 * 1000; // 4 minutes - private const int ConnectionMonitorInterval = 30 * 1000; // 30 seconds + /// + /// Gets the current health status of the connection pool. + /// + public ConnectionPoolHealth Health => GetHealthInternal(); public ImapClientPool(ImapClientPoolOptions imapClientPoolOptions) { _customServerInformation = imapClientPoolOptions.ServerInformation; _protocolLogStream = imapClientPoolOptions.ProtocolLog; - - // Set the maximum pool size to 5 or the custom value if it's greater. - _semaphore = new(Math.Max(MinimumPoolSize, _customServerInformation.MaxConcurrentClients)); - - CryptographyContext.Register(typeof(WindowsSecureMimeContext)); ImapClientPoolOptions = imapClientPoolOptions; - _keepAliveTimer = new System.Timers.Timer(KeepAliveInterval); - _connectionMonitorTimer = new System.Timers.Timer(ConnectionMonitorInterval); + CryptographyContext.Register(typeof(WindowsSecureMimeContext)); - _keepAliveTimer.Elapsed += KeepAliveTimerElapsed; - _connectionMonitorTimer.Elapsed += ConnectionMonitorTimerElapsed; + // Create unbounded channel for available clients + _availableClients = Channel.CreateUnbounded(new UnboundedChannelOptions + { + SingleReader = false, + SingleWriter = false, + AllowSynchronousContinuations = false + }); } - public async Task PreWarmPoolAsync() + /// + /// Initializes the pool by creating minimum connections and starting maintenance. + /// + public async Task InitializeAsync(CancellationToken cancellationToken = default) { + if (_initialized) return; + + _logger.Information("Initializing IMAP client pool with {MinConnections} connections", MinActiveConnections); + try { - for (int i = 0; i < MinimumPoolSize; i++) + // Create initial connections + for (int i = 0; i < MinActiveConnections; i++) { - var client = CreateNewClient(); - await EnsureCapabilitiesAsync(client, true); - _clients.Push(client); + cancellationToken.ThrowIfCancellationRequested(); + + var client = await CreateAndConnectClientAsync(cancellationToken).ConfigureAwait(false); + if (client != null) + { + _clientStates[client] = ImapClientState.Available; + await _availableClients.Writer.WriteAsync(client, cancellationToken).ConfigureAwait(false); + } } - // Start monitoring timers after pool is warmed - _keepAliveTimer.Start(); - _connectionMonitorTimer.Start(); + // Create dedicated IDLE client + _dedicatedIdleClient = await CreateAndConnectClientAsync(cancellationToken).ConfigureAwait(false); + if (_dedicatedIdleClient != null) + { + _clientStates[_dedicatedIdleClient] = ImapClientState.Idle; + } + + // Start maintenance task + _maintenanceTask = Task.Run(() => MaintenanceLoopAsync(_maintenanceCts.Token), _maintenanceCts.Token); + + _initialized = true; + _logger.Information("IMAP client pool initialized. Health: {Health}", Health.Summary); } catch (Exception ex) { - _logger.Error(ex, "Failed to pre-warm client pool"); - } - } - - private async void KeepAliveTimerElapsed(object sender, ElapsedEventArgs e) - { - foreach (var client in _clients) - { - try - { - if (client.IsConnected && !((WinoImapClient)client).IsBusy()) - { - await SendNoOpAsync(client); - } - } - catch (Exception ex) - { - _logger.Warning(ex, "Failed to send NOOP to client"); - } - } - } - - private async void ConnectionMonitorTimerElapsed(object sender, ElapsedEventArgs e) - { - foreach (var client in _clients) - { - try - { - if (!client.IsConnected && !((WinoImapClient)client).IsBusy()) - { - await EnsureCapabilitiesAsync(client, false); - } - } - catch (Exception ex) - { - _logger.Warning(ex, "Failed to reconnect client"); - } - } - } - - private async Task SendNoOpAsync(IImapClient client) - { - try - { - await client.NoOpAsync(); - } - catch (Exception ex) - { - _logger.Warning(ex, "NOOP command failed"); + _logger.Error(ex, "Failed to initialize IMAP client pool"); + throw; } } /// - /// Ensures all supported capabilities are enabled in this connection. - /// Reconnects and reauthenticates if necessary. - /// - /// Whether the client has been newly created. - private async Task EnsureCapabilitiesAsync(IImapClient client, bool isCreatedNew) + /// Pre-warms the pool (legacy compatibility method). + /// + public Task PreWarmPoolAsync() => InitializeAsync(CancellationToken.None); + + /// + /// Rents a client from the pool. Blocks until a client is available. + /// + public async Task RentAsync(CancellationToken cancellationToken = default) { - try + if (!_initialized) + await InitializeAsync(cancellationToken).ConfigureAwait(false); + + while (!cancellationToken.IsCancellationRequested) { - bool isReconnected = await EnsureConnectedAsync(client); - bool mustDoPostAuthIdentification = false; - - if ((isCreatedNew || isReconnected) && client.IsConnected) + // Try to get an available client from the channel + if (_availableClients.Reader.TryRead(out var client)) { - if (client.Capabilities.HasFlag(ImapCapabilities.Compress)) - await client.CompressAsync(); - - // Identify if the server supports ID extension. - // Some servers require it pre-authentication, some post-authentication. - // We'll observe the response here and do it after authentication if needed. - - if (client.Capabilities.HasFlag(ImapCapabilities.Id)) + if (client != null && _clientStates.TryGetValue(client, out var state) && state == ImapClientState.Available) { try { - await client.IdentifyAsync(_implementation); + // Ensure client is still connected + await EnsureClientReadyAsync(client, cancellationToken).ConfigureAwait(false); + _clientStates[client] = ImapClientState.InUse; + return client; } - catch (ImapCommandException commandException) when (commandException.Response == ImapCommandResponse.No || commandException.Response == ImapCommandResponse.Bad) + catch (Exception ex) { - mustDoPostAuthIdentification = true; - } - catch (Exception) - { - throw; + _logger.Warning(ex, "Client from pool was not ready, marking as failed"); + _clientStates[client] = ImapClientState.Failed; + // Continue to try next client or create new one } } } - await EnsureAuthenticatedAsync(client); - - if ((isCreatedNew || isReconnected) && client.IsAuthenticated) + // No available client, try to create a new one + var newClient = await CreateAndConnectClientAsync(cancellationToken).ConfigureAwait(false); + if (newClient != null) { - if (mustDoPostAuthIdentification) await client.IdentifyAsync(_implementation); + _clientStates[newClient] = ImapClientState.InUse; + return newClient; + } - // Activate post-auth capabilities. - if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync)) + // Wait a bit before retrying + await Task.Delay(100, cancellationToken).ConfigureAwait(false); + } + + throw new OperationCanceledException(cancellationToken); + } + + /// + /// Gets a client from the pool (legacy compatibility method). + /// + public async Task GetClientAsync() => await RentAsync(CancellationToken.None).ConfigureAwait(false); + + /// + /// Returns a client to the pool. + /// + public void Return(WinoImapClient client, bool isFaulted = false) + { + if (client == null) return; + + if (isFaulted || !client.IsConnected) + { + _clientStates[client] = ImapClientState.Failed; + DisposeClient(client); + return; + } + + if (!_disposedValue) + { + _clientStates[client] = ImapClientState.Available; + _availableClients.Writer.TryWrite(client); + } + else + { + DisposeClient(client); + } + } + + /// + /// Releases a client (legacy compatibility method). + /// + public void Release(IImapClient item, bool destroyClient = false) + { + if (item is WinoImapClient winoClient) + { + Return(winoClient, destroyClient); + } + else if (item != null) + { + DisposeClient(item); + } + } + + /// + /// Gets the dedicated IDLE client. Creates one if not available. + /// + public async Task GetIdleClientAsync(CancellationToken cancellationToken = default) + { + lock (_idleClientLock) + { + if (_dedicatedIdleClient != null && _dedicatedIdleClient.IsConnected) + { + return _dedicatedIdleClient; + } + } + + // Need to create or reconnect IDLE client + var idleClient = await CreateAndConnectClientAsync(cancellationToken).ConfigureAwait(false); + + lock (_idleClientLock) + { + if (_dedicatedIdleClient != null) + { + DisposeClient(_dedicatedIdleClient); + } + _dedicatedIdleClient = idleClient; + if (idleClient != null) + { + _clientStates[idleClient] = ImapClientState.Idle; + } + } + + return idleClient; + } + + /// + /// Releases the IDLE client for reconnection. + /// + public void ReleaseIdleClient(bool isFaulted = false) + { + lock (_idleClientLock) + { + if (_dedicatedIdleClient != null) + { + if (isFaulted) { - await client.EnableQuickResyncAsync().ConfigureAwait(false); - if (client is WinoImapClient winoImapClient) winoImapClient.IsQResyncEnabled = true; + _clientStates[_dedicatedIdleClient] = ImapClientState.Failed; + DisposeClient(_dedicatedIdleClient); + _dedicatedIdleClient = null; + } + else + { + _clientStates[_dedicatedIdleClient] = ImapClientState.Idle; } } } + } + + private ConnectionPoolHealth GetHealthInternal() + { + var health = new ConnectionPoolHealth + { + LastHealthCheck = DateTime.UtcNow, + IdleConnectionActive = _dedicatedIdleClient?.IsConnected ?? false + }; + + foreach (var kvp in _clientStates) + { + health.TotalConnections++; + switch (kvp.Value) + { + case ImapClientState.Available: + health.AvailableConnections++; + break; + case ImapClientState.InUse: + health.InUseConnections++; + break; + case ImapClientState.Failed: + health.FailedConnections++; + break; + case ImapClientState.Reconnecting: + health.ReconnectingConnections++; + break; + } + } + + return health; + } + + private async Task MaintenanceLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await Task.Delay(MaintenanceIntervalMs, cancellationToken).ConfigureAwait(false); + + // Send NOOP to keep connections alive + await SendNoOpToAvailableClientsAsync(cancellationToken).ConfigureAwait(false); + + // Ensure minimum connections + await EnsureMinimumConnectionsAsync(cancellationToken).ConfigureAwait(false); + + // Clean up failed connections + await CleanupFailedConnectionsAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.Warning(ex, "Error in pool maintenance loop"); + } + } + } + + private async Task SendNoOpToAvailableClientsAsync(CancellationToken cancellationToken) + { + foreach (var kvp in _clientStates) + { + if (kvp.Value == ImapClientState.Available && kvp.Key.IsConnected && !kvp.Key.IsBusy()) + { + try + { + await kvp.Key.NoOpAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.Debug(ex, "NOOP failed for client, marking as failed"); + _clientStates[kvp.Key] = ImapClientState.Failed; + } + } + } + } + + private async Task EnsureMinimumConnectionsAsync(CancellationToken cancellationToken) + { + var health = Health; + var neededConnections = MinActiveConnections - health.AvailableConnections; + + if (neededConnections > 0) + { + _logger.Debug("Creating {Count} connections to maintain minimum pool size", neededConnections); + + for (int i = 0; i < neededConnections; i++) + { + try + { + var client = await CreateAndConnectClientAsync(cancellationToken).ConfigureAwait(false); + if (client != null) + { + _clientStates[client] = ImapClientState.Available; + await _availableClients.Writer.WriteAsync(client, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.Warning(ex, "Failed to create new connection during maintenance"); + } + } + } + } + + private Task CleanupFailedConnectionsAsync(CancellationToken cancellationToken) + { + foreach (var kvp in _clientStates) + { + if (kvp.Value == ImapClientState.Failed) + { + DisposeClient(kvp.Key); + _clientStates.TryRemove(kvp.Key, out _); + } + } + return Task.CompletedTask; + } + + private async Task CreateAndConnectClientAsync(CancellationToken cancellationToken) + { + var client = CreateNewClient(); + + try + { + await EnsureClientReadyAsync(client, cancellationToken).ConfigureAwait(false); + return client; + } catch (Exception ex) { - if (ex.InnerException is ImapTestSSLCertificateException imapTestSSLCertificateException) - throw imapTestSSLCertificateException; + _logger.Warning(ex, "Failed to create and connect new client"); + DisposeClient(client); + return null; + } + } - throw new ImapClientPoolException(ex, GetProtocolLogContent()); - } - finally + private async Task EnsureClientReadyAsync(WinoImapClient client, CancellationToken cancellationToken) + { + // Connect if needed + if (!client.IsConnected) { - // Release it even if it fails. - _semaphore.Release(); + client.ServerCertificateValidationCallback = MyServerCertificateValidationCallback; + + await client.ConnectAsync( + _customServerInformation.IncomingServer, + int.Parse(_customServerInformation.IncomingServerPort), + GetSocketOptions(_customServerInformation.IncomingServerSocketOption), + cancellationToken).ConfigureAwait(false); + + // Enable compression if supported + if (client.Capabilities.HasFlag(ImapCapabilities.Compress)) + { + await client.CompressAsync(cancellationToken).ConfigureAwait(false); + } + + // Handle ID extension + if (client.Capabilities.HasFlag(ImapCapabilities.Id)) + { + try + { + await client.IdentifyAsync(_implementation, cancellationToken).ConfigureAwait(false); + } + catch (ImapCommandException) + { + // Some servers require post-auth identification + } + } } + + // Authenticate if needed + if (!client.IsAuthenticated) + { + var cred = new NetworkCredential( + _customServerInformation.IncomingServerUsername, + _customServerInformation.IncomingServerPassword); + + var authMethod = _customServerInformation.IncomingAuthenticationMethod; + + if (authMethod != ImapAuthenticationMethod.Auto) + { + client.AuthenticationMechanisms.Clear(); + var saslMechanism = GetSASLAuthenticationMethodName(authMethod); + client.AuthenticationMechanisms.Add(saslMechanism); + await client.AuthenticateAsync(SaslMechanism.Create(saslMechanism, cred), cancellationToken).ConfigureAwait(false); + } + else + { + await client.AuthenticateAsync(cred, cancellationToken).ConfigureAwait(false); + } + + // Try post-auth ID if needed + if (client.Capabilities.HasFlag(ImapCapabilities.Id)) + { + try + { + await client.IdentifyAsync(_implementation, cancellationToken).ConfigureAwait(false); + } + catch { /* Ignore */ } + } + + // Enable QRESYNC if supported + if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync)) + { + await client.EnableQuickResyncAsync(cancellationToken).ConfigureAwait(false); + client.IsQResyncEnabled = true; + } + } + } + + private WinoImapClient CreateNewClient() + { + var client = new WinoImapClient(); + + if (!string.IsNullOrEmpty(_customServerInformation.ProxyServer)) + { + client.ProxyClient = new HttpProxyClient( + _customServerInformation.ProxyServer, + int.Parse(_customServerInformation.ProxyServerPort)); + } + + _logger.Debug("Created new ImapClient. Current pool size: {Count}", _clientStates.Count); + return client; + } + + private void DisposeClient(IImapClient client) + { + try + { + if (client.IsConnected) + { + lock (client.SyncRoot) + { + client.Disconnect(quit: true); + } + } + client.Dispose(); + } + catch (Exception ex) + { + _logger.Debug(ex, "Error disposing client"); + } + } + + private SecureSocketOptions GetSocketOptions(ImapConnectionSecurity connectionSecurity) => connectionSecurity switch + { + ImapConnectionSecurity.Auto => SecureSocketOptions.Auto, + ImapConnectionSecurity.None => SecureSocketOptions.None, + ImapConnectionSecurity.StartTls => SecureSocketOptions.StartTlsWhenAvailable, + ImapConnectionSecurity.SslTls => SecureSocketOptions.SslOnConnect, + _ => SecureSocketOptions.None + }; + + private string GetSASLAuthenticationMethodName(ImapAuthenticationMethod method) => method switch + { + ImapAuthenticationMethod.NormalPassword => "PLAIN", + ImapAuthenticationMethod.EncryptedPassword => "LOGIN", + ImapAuthenticationMethod.Ntlm => "NTLM", + ImapAuthenticationMethod.CramMd5 => "CRAM-MD5", + ImapAuthenticationMethod.DigestMd5 => "DIGEST-MD5", + _ => "PLAIN" + }; + + private bool MyServerCertificateValidationCallback(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) + { + if (sslPolicyErrors == SslPolicyErrors.None) return true; + + if (ThrowOnSSLHandshakeCallback) + { + throw new ImapTestSSLCertificateException( + certificate.Issuer, + certificate.GetExpirationDateString(), + certificate.GetEffectiveDateString()); + } + + return true; } public string GetProtocolLogContent() { if (_protocolLogStream == null) return default; - // Set the position to the beginning of the stream in case it is not already at the start if (_protocolLogStream.CanSeek) _protocolLogStream.Seek(0, SeekOrigin.Begin); @@ -223,171 +575,12 @@ public class ImapClientPool : IDisposable return reader.ReadToEnd(); } - public async Task GetClientAsync() - { - await _semaphore.WaitAsync(); + // Legacy compatibility methods + public Task EnsureConnectedAsync(IImapClient client) => + Task.FromResult(client.IsConnected); - if (_clients.TryPop(out IImapClient item)) - { - await EnsureCapabilitiesAsync(item, false); - - return item; - } - - var client = CreateNewClient(); - - await EnsureCapabilitiesAsync(client, true); - - return client; - } - - public void Release(IImapClient item, bool destroyClient = false) - { - if (item != null) - { - if (destroyClient) - { - if (item.IsConnected) - { - lock (item.SyncRoot) - { - item.Disconnect(quit: true); - } - } - item.Dispose(); - } - else if (!_disposedValue) - { - _clients.Push(item); - } - - _semaphore.Release(); - } - } - - private IImapClient CreateNewClient() - { - WinoImapClient client = null; - - client = new WinoImapClient(); - - HttpProxyClient proxyClient = null; - - // Add proxy client if exists. - if (!string.IsNullOrEmpty(_customServerInformation.ProxyServer)) - { - proxyClient = new HttpProxyClient(_customServerInformation.ProxyServer, int.Parse(_customServerInformation.ProxyServerPort)); - client.ProxyClient = proxyClient; - } - - _logger.Debug("Creating new ImapClient. Current clients: {Count}", _clients.Count); - - return client; - } - - private SecureSocketOptions GetSocketOptions(ImapConnectionSecurity connectionSecurity) - => connectionSecurity switch - { - ImapConnectionSecurity.Auto => SecureSocketOptions.Auto, - ImapConnectionSecurity.None => SecureSocketOptions.None, - ImapConnectionSecurity.StartTls => SecureSocketOptions.StartTlsWhenAvailable, - ImapConnectionSecurity.SslTls => SecureSocketOptions.SslOnConnect, - _ => SecureSocketOptions.None - }; - - /// True if the connection is newly established. - public async Task EnsureConnectedAsync(IImapClient client) - { - if (client.IsConnected) return false; - - client.ServerCertificateValidationCallback = MyServerCertificateValidationCallback; - - await client.ConnectAsync(_customServerInformation.IncomingServer, - int.Parse(_customServerInformation.IncomingServerPort), - GetSocketOptions(_customServerInformation.IncomingServerSocketOption)); - - // Print out useful information for testing. - if (client.IsConnected && ImapClientPoolOptions.IsTestPool) - { - // Print supported authentication methods for the client. - var supportedAuthMethods = client.AuthenticationMechanisms; - - if (supportedAuthMethods == null || supportedAuthMethods.Count == 0) - { - WriteToProtocolLog("There are no supported authentication mechanisms..."); - } - else - { - WriteToProtocolLog($"Supported authentication mechanisms: {string.Join(", ", supportedAuthMethods)}"); - } - } - - return true; - } - - public async Task EnsureAuthenticatedAsync(IImapClient client) - { - if (client.IsAuthenticated) return; - - var cred = new NetworkCredential(_customServerInformation.IncomingServerUsername, _customServerInformation.IncomingServerPassword); - var prefferedAuthenticationMethod = _customServerInformation.IncomingAuthenticationMethod; - - if (prefferedAuthenticationMethod != ImapAuthenticationMethod.Auto) - { - // Anything beside Auto must be explicitly set for the client. - client.AuthenticationMechanisms.Clear(); - var saslMechanism = GetSASLAuthenticationMethodName(prefferedAuthenticationMethod); - client.AuthenticationMechanisms.Add(saslMechanism); - await client.AuthenticateAsync(SaslMechanism.Create(saslMechanism, cred)); - } - else - { - await client.AuthenticateAsync(cred); - } - } - - private string GetSASLAuthenticationMethodName(ImapAuthenticationMethod method) - => method switch - { - ImapAuthenticationMethod.NormalPassword => "PLAIN", - ImapAuthenticationMethod.EncryptedPassword => "LOGIN", - ImapAuthenticationMethod.Ntlm => "NTLM", - ImapAuthenticationMethod.CramMd5 => "CRAM-MD5", - ImapAuthenticationMethod.DigestMd5 => "DIGEST-MD5", - _ => "PLAIN" - }; - - private void WriteToProtocolLog(string message) - { - if (_protocolLogStream == null) return; - try - { - var messageBytes = Encoding.UTF8.GetBytes($"W: {message}\n"); - _protocolLogStream.Write(messageBytes, 0, messageBytes.Length); - } - catch (ObjectDisposedException) - { - Log.Warning($"Protocol log stream is disposed. Cannot write to it."); - } - catch (Exception) - { - throw; - } - } - - bool MyServerCertificateValidationCallback(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) - { - // If there are no errors, then everything went smoothly. - if (sslPolicyErrors == SslPolicyErrors.None) return true; - - // Imap connectivity test will throw to alert the user here. - if (ThrowOnSSLHandshakeCallback) - { - throw new ImapTestSSLCertificateException(certificate.Issuer, certificate.GetExpirationDateString(), certificate.GetEffectiveDateString()); - } - - return true; - } + public Task EnsureAuthenticatedAsync(IImapClient client) => + Task.CompletedTask; protected virtual void Dispose(bool disposing) { @@ -395,22 +588,26 @@ public class ImapClientPool : IDisposable { if (disposing) { - _keepAliveTimer.Stop(); - _connectionMonitorTimer.Stop(); + _maintenanceCts.Cancel(); + _maintenanceTask?.Wait(TimeSpan.FromSeconds(5)); + _maintenanceCts.Dispose(); - _keepAliveTimer.Dispose(); - _connectionMonitorTimer.Dispose(); + _availableClients.Writer.Complete(); - _clients.ForEach(client => + foreach (var kvp in _clientStates) { - lock (client.SyncRoot) - { - client.Disconnect(true); - } - client.Dispose(); - }); + DisposeClient(kvp.Key); + } + _clientStates.Clear(); - _clients.Clear(); + lock (_idleClientLock) + { + if (_dedicatedIdleClient != null) + { + DisposeClient(_dedicatedIdleClient); + _dedicatedIdleClient = null; + } + } } _disposedValue = true; diff --git a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs index 3c142f3e..abf6c21a 100644 --- a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs @@ -107,6 +107,13 @@ public interface IImapChangeProcessor : IDefaultChangeProcessor /// /// Folder id to retrieve uIds for. Task> GetKnownUidsForFolderAsync(Guid folderId); + + /// + /// Gets the most recent mail IDs for a folder (for notification purposes). + /// + /// Folder ID. + /// Number of recent mails to return. + Task> GetRecentMailIdsForFolderAsync(Guid folderId, int count); } public class DefaultChangeProcessor(IDatabaseService databaseService, diff --git a/Wino.Core/Integration/Processors/ImapChangeProcessor.cs b/Wino.Core/Integration/Processors/ImapChangeProcessor.cs index f23470bf..72b74f04 100644 --- a/Wino.Core/Integration/Processors/ImapChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/ImapChangeProcessor.cs @@ -18,4 +18,7 @@ public class ImapChangeProcessor : DefaultChangeProcessor, IImapChangeProcessor } public Task> GetKnownUidsForFolderAsync(Guid folderId) => FolderService.GetKnownUidsForFolderAsync(folderId); + + public Task> GetRecentMailIdsForFolderAsync(Guid folderId, int count) + => MailService.GetRecentMailIdsForFolderAsync(folderId, count); } diff --git a/Wino.Core/Integration/WinoImapClient.cs b/Wino.Core/Integration/WinoImapClient.cs index f4712268..c5d80057 100644 --- a/Wino.Core/Integration/WinoImapClient.cs +++ b/Wino.Core/Integration/WinoImapClient.cs @@ -9,7 +9,7 @@ namespace Wino.Core.Integration; /// /// Extended class for ImapClient that is used in Wino. /// -internal class WinoImapClient : ImapClient +public class WinoImapClient : ImapClient { private int _busyCount; diff --git a/Wino.Core/Services/GmailSynchronizerErrorHandlingFactory.cs b/Wino.Core/Services/GmailSynchronizerErrorHandlingFactory.cs index b2153c32..0abc7372 100644 --- a/Wino.Core/Services/GmailSynchronizerErrorHandlingFactory.cs +++ b/Wino.Core/Services/GmailSynchronizerErrorHandlingFactory.cs @@ -1,11 +1,22 @@ -using System.Threading.Tasks; -using Wino.Core.Domain.Interfaces; -using Wino.Core.Domain.Models.Errors; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Synchronizers.Errors.Gmail; namespace Wino.Core.Services; + +/// +/// Factory for handling Gmail synchronizer errors. +/// Registers and routes errors to appropriate handlers. +/// public class GmailSynchronizerErrorHandlingFactory : SynchronizerErrorHandlingFactory, IGmailSynchronizerErrorHandlerFactory { - public bool CanHandle(SynchronizerErrorContext error) => CanHandle(error); - - public Task HandleAsync(SynchronizerErrorContext error) => HandleErrorAsync(error); + public GmailSynchronizerErrorHandlingFactory( + GmailQuotaExceededHandler quotaExceededHandler, + GmailRateLimitHandler rateLimitHandler, + GmailHistoryExpiredHandler historyExpiredHandler) + { + // Order matters - more specific handlers should be registered first + RegisterHandler(quotaExceededHandler); + RegisterHandler(historyExpiredHandler); + RegisterHandler(rateLimitHandler); // Most generic rate limit handler last + } } diff --git a/Wino.Core/Services/ImapSynchronizerErrorHandlingFactory.cs b/Wino.Core/Services/ImapSynchronizerErrorHandlingFactory.cs new file mode 100644 index 00000000..662058c7 --- /dev/null +++ b/Wino.Core/Services/ImapSynchronizerErrorHandlingFactory.cs @@ -0,0 +1,24 @@ +using Wino.Core.Domain.Interfaces; +using Wino.Core.Synchronizers.Errors.Imap; + +namespace Wino.Core.Services; + +/// +/// Factory for handling IMAP synchronizer errors. +/// Registers and routes errors to appropriate handlers. +/// +public class ImapSynchronizerErrorHandlingFactory : SynchronizerErrorHandlingFactory, IImapSynchronizerErrorHandlerFactory +{ + public ImapSynchronizerErrorHandlingFactory( + ImapConnectionLostHandler connectionLostHandler, + ImapAuthenticationFailedHandler authFailedHandler, + ImapFolderNotFoundHandler folderNotFoundHandler, + ImapProtocolErrorHandler protocolErrorHandler) + { + // Order matters - more specific handlers should be registered first + RegisterHandler(authFailedHandler); + RegisterHandler(folderNotFoundHandler); + RegisterHandler(connectionLostHandler); + RegisterHandler(protocolErrorHandler); // Most generic, registered last + } +} diff --git a/Wino.Core/Services/OutlookSynchronizerErrorHandlingFactory.cs b/Wino.Core/Services/OutlookSynchronizerErrorHandlingFactory.cs index 437a331b..2767ecd9 100644 --- a/Wino.Core/Services/OutlookSynchronizerErrorHandlingFactory.cs +++ b/Wino.Core/Services/OutlookSynchronizerErrorHandlingFactory.cs @@ -1,6 +1,6 @@ using System.Threading.Tasks; using Wino.Core.Domain.Interfaces; -using Wino.Core.Domain.Models.Errors; +using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Synchronizers.Errors.Outlook; namespace Wino.Core.Services; diff --git a/Wino.Core/Services/RetryExecutor.cs b/Wino.Core/Services/RetryExecutor.cs new file mode 100644 index 00000000..f59d5ffb --- /dev/null +++ b/Wino.Core/Services/RetryExecutor.cs @@ -0,0 +1,140 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Retry; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Services; + +/// +/// Executes operations with automatic retry and error handling support. +/// Implements exponential backoff with jitter. +/// +public class RetryExecutor : IRetryExecutor +{ + private readonly ILogger _logger = Log.ForContext(); + + /// + public async Task ExecuteWithRetryAsync( + Func> operation, + RetryPolicy policy, + Func errorContextFactory, + ISynchronizerErrorHandlerFactory errorHandler = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(policy); + ArgumentNullException.ThrowIfNull(errorContextFactory); + + int attempt = 0; + Exception lastException = null; + + while (attempt <= policy.MaxRetries) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + return await operation(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; // Don't retry on cancellation + } + catch (Exception ex) + { + lastException = ex; + attempt++; + + var errorContext = errorContextFactory(ex); + errorContext.RetryCount = attempt; + errorContext.MaxRetries = policy.MaxRetries; + + // Let the error handler process the error first + if (errorHandler != null) + { + try + { + var handled = await errorHandler.HandleErrorAsync(errorContext).ConfigureAwait(false); + if (handled) + { + _logger.Debug("Error handled by error handler, severity: {Severity}", errorContext.Severity); + } + } + catch (Exception handlerEx) + { + _logger.Warning(handlerEx, "Error handler threw an exception"); + } + } + + // Check if we should retry based on error severity + if (errorContext.Severity == SynchronizerErrorSeverity.Fatal || + errorContext.Severity == SynchronizerErrorSeverity.AuthRequired) + { + _logger.Warning(ex, "Non-retryable error (severity: {Severity}), failing immediately", errorContext.Severity); + throw; + } + + if (errorContext.Severity == SynchronizerErrorSeverity.Recoverable) + { + _logger.Debug(ex, "Recoverable error, not retrying but allowing continuation"); + throw; + } + + // Transient error - check if we have retries left + if (attempt > policy.MaxRetries) + { + _logger.Warning(ex, "All {MaxRetries} retries exhausted", policy.MaxRetries); + throw; + } + + // Calculate delay and wait + var delay = errorContext.RetryDelay ?? policy.GetDelay(attempt); + _logger.Debug("Retry attempt {Attempt}/{MaxRetries} after {Delay}ms delay for error: {ErrorMessage}", + attempt, policy.MaxRetries, delay.TotalMilliseconds, ex.Message); + + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + + // Should not reach here, but just in case + throw lastException ?? new InvalidOperationException("Retry loop completed without result"); + } + + /// + public async Task ExecuteWithRetryAsync( + Func operation, + RetryPolicy policy, + Func errorContextFactory, + ISynchronizerErrorHandlerFactory errorHandler = null, + CancellationToken cancellationToken = default) + { + await ExecuteWithRetryAsync( + async ct => + { + await operation(ct).ConfigureAwait(false); + return true; // Dummy return value + }, + policy, + errorContextFactory, + errorHandler, + cancellationToken).ConfigureAwait(false); + } + + /// + public Task ExecuteWithRetryAsync( + Func> operation, + Func errorContextFactory, + CancellationToken cancellationToken = default) + { + return ExecuteWithRetryAsync( + operation, + RetryPolicy.Default, + errorContextFactory, + null, + cancellationToken); + } +} diff --git a/Wino.Core/Services/SynchronizerErrorHandlingFactory.cs b/Wino.Core/Services/SynchronizerErrorHandlingFactory.cs index 7c9336c0..446688db 100644 --- a/Wino.Core/Services/SynchronizerErrorHandlingFactory.cs +++ b/Wino.Core/Services/SynchronizerErrorHandlingFactory.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Serilog; using Wino.Core.Domain.Interfaces; -using Wino.Core.Domain.Models.Errors; +using Wino.Core.Domain.Models.Synchronization; namespace Wino.Core.Services; diff --git a/Wino.Core/Services/SynchronizerFactory.cs b/Wino.Core/Services/SynchronizerFactory.cs index 51865487..4a80c5cc 100644 --- a/Wino.Core/Services/SynchronizerFactory.cs +++ b/Wino.Core/Services/SynchronizerFactory.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Wino.Core.Domain.Entities.Shared; using Wino.Core.Domain.Interfaces; using Wino.Core.Integration.Processors; +using Wino.Core.Synchronizers.ImapSync; using Wino.Core.Synchronizers.Mail; namespace Wino.Core.Services; @@ -17,10 +18,12 @@ public class SynchronizerFactory : ISynchronizerFactory private readonly IApplicationConfiguration _applicationConfiguration; private readonly IOutlookSynchronizerErrorHandlerFactory _outlookSynchronizerErrorHandlerFactory; private readonly IGmailSynchronizerErrorHandlerFactory _gmailSynchronizerErrorHandlerFactory; + private readonly IImapSynchronizerErrorHandlerFactory _imapSynchronizerErrorHandlerFactory; private readonly IOutlookChangeProcessor _outlookChangeProcessor; private readonly IGmailChangeProcessor _gmailChangeProcessor; private readonly IImapChangeProcessor _imapChangeProcessor; private readonly IAuthenticationProvider _authenticationProvider; + private readonly UnifiedImapSynchronizer _unifiedImapSynchronizer; private readonly List synchronizerCache = new(); @@ -32,7 +35,9 @@ public class SynchronizerFactory : ISynchronizerFactory IImapSynchronizationStrategyProvider imapSynchronizationStrategyProvider, IApplicationConfiguration applicationConfiguration, IOutlookSynchronizerErrorHandlerFactory outlookSynchronizerErrorHandlerFactory, - IGmailSynchronizerErrorHandlerFactory gmailSynchronizerErrorHandlerFactory) + IGmailSynchronizerErrorHandlerFactory gmailSynchronizerErrorHandlerFactory, + IImapSynchronizerErrorHandlerFactory imapSynchronizerErrorHandlerFactory, + UnifiedImapSynchronizer unifiedImapSynchronizer) { _outlookChangeProcessor = outlookChangeProcessor; _gmailChangeProcessor = gmailChangeProcessor; @@ -43,6 +48,8 @@ public class SynchronizerFactory : ISynchronizerFactory _applicationConfiguration = applicationConfiguration; _outlookSynchronizerErrorHandlerFactory = outlookSynchronizerErrorHandlerFactory; _gmailSynchronizerErrorHandlerFactory = gmailSynchronizerErrorHandlerFactory; + _imapSynchronizerErrorHandlerFactory = imapSynchronizerErrorHandlerFactory; + _unifiedImapSynchronizer = unifiedImapSynchronizer; } public async Task GetAccountSynchronizerAsync(Guid accountId) @@ -78,7 +85,7 @@ public class SynchronizerFactory : ISynchronizerFactory var gmailAuthenticator = _authenticationProvider.GetAuthenticator(Domain.Enums.MailProviderType.Gmail) as IGmailAuthenticator; return new GmailSynchronizer(mailAccount, gmailAuthenticator, _gmailChangeProcessor, _gmailSynchronizerErrorHandlerFactory); case Domain.Enums.MailProviderType.IMAP4: - return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _imapSynchronizationStrategyProvider, _applicationConfiguration); + return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _imapSynchronizationStrategyProvider, _applicationConfiguration, _unifiedImapSynchronizer, _imapSynchronizerErrorHandlerFactory); default: break; } diff --git a/Wino.Core/Synchronizers/Errors/Gmail/GmailHistoryExpiredHandler.cs b/Wino.Core/Synchronizers/Errors/Gmail/GmailHistoryExpiredHandler.cs new file mode 100644 index 00000000..70435125 --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Gmail/GmailHistoryExpiredHandler.cs @@ -0,0 +1,78 @@ +using System; +using System.Threading.Tasks; +using Google; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; +using Wino.Core.Integration.Processors; + +namespace Wino.Core.Synchronizers.Errors.Gmail; + +/// +/// Handles Gmail history ID expiration errors. +/// When history is no longer available, resets the account's history ID to force a full resync. +/// +public class GmailHistoryExpiredHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + private readonly IGmailChangeProcessor _gmailChangeProcessor; + + public GmailHistoryExpiredHandler(IGmailChangeProcessor gmailChangeProcessor) + { + _gmailChangeProcessor = gmailChangeProcessor; + } + + public bool CanHandle(SynchronizerErrorContext error) + { + // Gmail returns 404 when history ID is no longer valid + if (error.ErrorCode == 404) + { + var message = error.ErrorMessage?.ToLowerInvariant() ?? string.Empty; + return message.Contains("history") || message.Contains("notfound"); + } + + if (error.Exception is GoogleApiException googleEx) + { + if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.NotFound) + { + var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty; + return errorMessage.Contains("history") || + errorMessage.Contains("not found") || + errorMessage.Contains("starthistoryid"); + } + } + + return false; + } + + public async Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "Gmail history ID expired for account {AccountName} ({AccountId}). Resetting to force full sync.", + error.Account?.Name, error.Account?.Id); + + error.Severity = SynchronizerErrorSeverity.Recoverable; + error.Category = SynchronizerErrorCategory.ResourceNotFound; + + // Reset the account's synchronization identifier (history ID) + if (error.Account != null) + { + try + { + await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync( + error.Account.Id, string.Empty).ConfigureAwait(false); + + _logger.Information("Successfully reset Gmail history ID for account {AccountName}. Next sync will be full sync.", + error.Account.Name); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to reset Gmail history ID for account {AccountName}", + error.Account.Name); + } + } + + return true; + } +} diff --git a/Wino.Core/Synchronizers/Errors/Gmail/GmailQuotaExceededHandler.cs b/Wino.Core/Synchronizers/Errors/Gmail/GmailQuotaExceededHandler.cs new file mode 100644 index 00000000..77363d4a --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Gmail/GmailQuotaExceededHandler.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Google; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Synchronizers.Errors.Gmail; + +/// +/// Handles Gmail API quota exceeded errors (HTTP 403 with quota error). +/// This is a more severe rate limit that indicates daily quota exhaustion. +/// +public class GmailQuotaExceededHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + + public bool CanHandle(SynchronizerErrorContext error) + { + if (error.Exception is GoogleApiException googleEx) + { + // Quota exceeded usually returns 403 + if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.Forbidden) + { + var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty; + var errorReason = googleEx.Error?.Errors?.FirstOrDefault()?.Reason?.ToLowerInvariant() ?? string.Empty; + + return errorMessage.Contains("quota") || + errorMessage.Contains("limit exceeded") || + errorReason.Contains("quota") || + errorReason.Contains("ratelimitexceeded") || + errorReason.Contains("userlimitexceeded"); + } + } + + return false; + } + + public Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "Gmail API quota exceeded for account {AccountName} ({AccountId}). Sync will be paused.", + error.Account?.Name, error.Account?.Id); + + // Quota exceeded is more severe - treat as fatal to prevent repeated failures + // The user will be notified and sync will resume after quota resets + error.Severity = SynchronizerErrorSeverity.Fatal; + error.Category = SynchronizerErrorCategory.RateLimit; + + // Suggest a very long delay - quotas typically reset daily + error.RetryDelay = TimeSpan.FromHours(1); + + return Task.FromResult(true); + } +} diff --git a/Wino.Core/Synchronizers/Errors/Gmail/GmailRateLimitHandler.cs b/Wino.Core/Synchronizers/Errors/Gmail/GmailRateLimitHandler.cs new file mode 100644 index 00000000..30251513 --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Gmail/GmailRateLimitHandler.cs @@ -0,0 +1,47 @@ +using System; +using System.Threading.Tasks; +using Google; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Synchronizers.Errors.Gmail; + +/// +/// Handles Gmail API rate limiting errors (HTTP 429 Too Many Requests). +/// Marks the error as transient with appropriate backoff delay. +/// +public class GmailRateLimitHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + + public bool CanHandle(SynchronizerErrorContext error) + { + if (error.ErrorCode == 429) + return true; + + if (error.Exception is GoogleApiException googleEx) + { + return googleEx.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests || + (googleEx.Error?.Code == 429); + } + + return false; + } + + public Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "Gmail API rate limit hit for account {AccountName} ({AccountId}). Operation: {Operation}. Will retry with backoff.", + error.Account?.Name, error.Account?.Id, error.OperationType ?? "N/A"); + + error.Severity = SynchronizerErrorSeverity.Transient; + error.Category = SynchronizerErrorCategory.RateLimit; + + // Gmail rate limits are usually per-user, suggest a longer delay + error.RetryDelay = TimeSpan.FromSeconds(10); + + return Task.FromResult(true); + } +} diff --git a/Wino.Core/Synchronizers/Errors/Imap/ImapAuthenticationFailedHandler.cs b/Wino.Core/Synchronizers/Errors/Imap/ImapAuthenticationFailedHandler.cs new file mode 100644 index 00000000..d86a218e --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Imap/ImapAuthenticationFailedHandler.cs @@ -0,0 +1,40 @@ +using System.Threading.Tasks; +using MailKit.Security; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Synchronizers.Errors.Imap; + +/// +/// Handles IMAP authentication failures (AuthenticationException, SaslException). +/// Marks the error as requiring re-authentication. +/// +public class ImapAuthenticationFailedHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + + public bool CanHandle(SynchronizerErrorContext error) + { + return error.Exception is AuthenticationException || + error.Exception is SaslException || + (error.ErrorMessage?.Contains("authentication", System.StringComparison.OrdinalIgnoreCase) ?? false); + } + + public Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "IMAP authentication failed for account {AccountName} ({AccountId}). User needs to re-authenticate.", + error.Account?.Name, error.Account?.Id); + + // Mark as requiring authentication - this will stop sync and notify user + error.Severity = SynchronizerErrorSeverity.AuthRequired; + error.Category = SynchronizerErrorCategory.Authentication; + + // No point in retrying auth failures - credentials need to be updated + error.RetryDelay = null; + + return Task.FromResult(true); + } +} diff --git a/Wino.Core/Synchronizers/Errors/Imap/ImapConnectionLostHandler.cs b/Wino.Core/Synchronizers/Errors/Imap/ImapConnectionLostHandler.cs new file mode 100644 index 00000000..186dca26 --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Imap/ImapConnectionLostHandler.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; +using System.Net.Sockets; +using System.Threading.Tasks; +using MailKit; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Synchronizers.Errors.Imap; + +/// +/// Handles IMAP connection loss errors (IOException, SocketException, ServiceNotConnectedException). +/// Marks the error as transient for retry with backoff. +/// +public class ImapConnectionLostHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + + public bool CanHandle(SynchronizerErrorContext error) + { + return error.Exception is IOException || + error.Exception is SocketException || + error.Exception is ServiceNotConnectedException || + error.Exception?.InnerException is IOException || + error.Exception?.InnerException is SocketException; + } + + public Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "IMAP connection lost for account {AccountName} ({AccountId}). Folder: {FolderName}. Operation: {Operation}. Will retry.", + error.Account?.Name, error.Account?.Id, error.FolderName ?? "N/A", error.OperationType ?? "N/A"); + + // Mark as transient - the RetryExecutor will handle the retry logic + error.Severity = SynchronizerErrorSeverity.Transient; + error.Category = SynchronizerErrorCategory.Network; + + // Suggest a reasonable retry delay for connection issues + error.RetryDelay = TimeSpan.FromSeconds(2); + + return Task.FromResult(true); + } +} diff --git a/Wino.Core/Synchronizers/Errors/Imap/ImapFolderNotFoundHandler.cs b/Wino.Core/Synchronizers/Errors/Imap/ImapFolderNotFoundHandler.cs new file mode 100644 index 00000000..d23302c3 --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Imap/ImapFolderNotFoundHandler.cs @@ -0,0 +1,67 @@ +using System.Threading.Tasks; +using MailKit; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; +using Wino.Core.Integration.Processors; + +namespace Wino.Core.Synchronizers.Errors.Imap; + +/// +/// Handles IMAP folder not found errors (FolderNotFoundException). +/// Deletes the folder locally and allows sync to continue with other folders. +/// +public class ImapFolderNotFoundHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + private readonly IImapChangeProcessor _imapChangeProcessor; + + public ImapFolderNotFoundHandler(IImapChangeProcessor imapChangeProcessor) + { + _imapChangeProcessor = imapChangeProcessor; + } + + public bool CanHandle(SynchronizerErrorContext error) + { + return error.Exception is FolderNotFoundException || + error.ErrorCode == 404 || + (error.ErrorMessage?.Contains("folder not found", System.StringComparison.OrdinalIgnoreCase) ?? false) || + (error.ErrorMessage?.Contains("mailbox not found", System.StringComparison.OrdinalIgnoreCase) ?? false); + } + + public async Task HandleAsync(SynchronizerErrorContext error) + { + _logger.Warning(error.Exception, + "IMAP folder not found for account {AccountName} ({AccountId}). Folder: {FolderName} ({FolderId}). Removing locally.", + error.Account?.Name, error.Account?.Id, error.FolderName, error.FolderId); + + // Mark as recoverable - sync can continue with other folders + error.Severity = SynchronizerErrorSeverity.Recoverable; + error.Category = SynchronizerErrorCategory.ResourceNotFound; + + // Try to delete the folder locally if we have the folder ID + if (error.FolderId.HasValue && error.Account != null) + { + try + { + // Get the folder's remote ID from the exception if available + var remoteId = error.Exception is FolderNotFoundException fnf ? fnf.FolderName : null; + + if (!string.IsNullOrEmpty(remoteId)) + { + await _imapChangeProcessor.DeleteFolderAsync(error.Account.Id, remoteId).ConfigureAwait(false); + _logger.Information("Successfully deleted local folder {FolderName} after server deletion.", + error.FolderName); + } + } + catch (System.Exception ex) + { + _logger.Warning(ex, "Failed to delete local folder {FolderName} ({FolderId})", + error.FolderName, error.FolderId); + } + } + + return true; + } +} diff --git a/Wino.Core/Synchronizers/Errors/Imap/ImapProtocolErrorHandler.cs b/Wino.Core/Synchronizers/Errors/Imap/ImapProtocolErrorHandler.cs new file mode 100644 index 00000000..6d4eb540 --- /dev/null +++ b/Wino.Core/Synchronizers/Errors/Imap/ImapProtocolErrorHandler.cs @@ -0,0 +1,100 @@ +using System; +using System.Threading.Tasks; +using MailKit.Net.Imap; +using Serilog; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.Synchronization; + +namespace Wino.Core.Synchronizers.Errors.Imap; + +/// +/// Handles generic IMAP protocol errors (ImapProtocolException, ImapCommandException). +/// This is the catch-all handler for IMAP errors not handled by more specific handlers. +/// +public class ImapProtocolErrorHandler : ISynchronizerErrorHandler +{ + private readonly ILogger _logger = Log.ForContext(); + + public bool CanHandle(SynchronizerErrorContext error) + { + // This is a catch-all for IMAP-related exceptions + return error.Exception is ImapProtocolException || + error.Exception is ImapCommandException; + } + + public Task HandleAsync(SynchronizerErrorContext error) + { + var severity = ClassifyProtocolError(error); + var category = SynchronizerErrorCategory.ProtocolError; + + _logger.Warning(error.Exception, + "IMAP protocol error for account {AccountName} ({AccountId}). Folder: {FolderName}. Operation: {Operation}. Severity: {Severity}", + error.Account?.Name, error.Account?.Id, error.FolderName ?? "N/A", error.OperationType ?? "N/A", severity); + + error.Severity = severity; + error.Category = category; + + // For transient protocol errors, suggest a retry delay + if (severity == SynchronizerErrorSeverity.Transient) + { + error.RetryDelay = TimeSpan.FromSeconds(5); + } + + return Task.FromResult(true); + } + + /// + /// Classifies the protocol error to determine if it's transient, recoverable, or fatal. + /// + private static SynchronizerErrorSeverity ClassifyProtocolError(SynchronizerErrorContext error) + { + var message = error.ErrorMessage?.ToLowerInvariant() ?? string.Empty; + var exMessage = error.Exception?.Message?.ToLowerInvariant() ?? string.Empty; + + // Check for rate limiting / throttling + if (message.Contains("too many") || message.Contains("rate limit") || + message.Contains("throttl") || exMessage.Contains("too many")) + { + return SynchronizerErrorSeverity.Transient; + } + + // Check for temporary server issues + if (message.Contains("try again") || message.Contains("temporary") || + message.Contains("busy") || exMessage.Contains("try again")) + { + return SynchronizerErrorSeverity.Transient; + } + + // Check for command-specific errors that are usually transient + if (error.Exception is ImapCommandException cmdEx) + { + // NO response usually means the operation failed but can be retried + if (cmdEx.Response == ImapCommandResponse.No) + { + // Unless it's a permanent failure indication + if (message.Contains("permanent") || message.Contains("invalid")) + { + return SynchronizerErrorSeverity.Recoverable; + } + return SynchronizerErrorSeverity.Transient; + } + + // BAD response usually indicates a protocol violation - don't retry + if (cmdEx.Response == ImapCommandResponse.Bad) + { + return SynchronizerErrorSeverity.Recoverable; + } + } + + // Protocol exceptions that indicate connection issues + if (error.Exception is ImapProtocolException) + { + // Most protocol exceptions are connection-related and transient + return SynchronizerErrorSeverity.Transient; + } + + // Default to recoverable for unknown protocol errors + return SynchronizerErrorSeverity.Recoverable; + } +} diff --git a/Wino.Core/Synchronizers/Errors/Outlook/DeltaTokenExpiredHandler.cs b/Wino.Core/Synchronizers/Errors/Outlook/DeltaTokenExpiredHandler.cs index b62c690d..28a963b3 100644 --- a/Wino.Core/Synchronizers/Errors/Outlook/DeltaTokenExpiredHandler.cs +++ b/Wino.Core/Synchronizers/Errors/Outlook/DeltaTokenExpiredHandler.cs @@ -3,7 +3,7 @@ using Microsoft.Graph.Models.ODataErrors; using Microsoft.Kiota.Abstractions; using Serilog; using Wino.Core.Domain.Interfaces; -using Wino.Core.Domain.Models.Errors; +using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Integration.Processors; namespace Wino.Core.Synchronizers.Errors.Outlook; diff --git a/Wino.Core/Synchronizers/Errors/Outlook/ObjectCannotBeDeletedHandler.cs b/Wino.Core/Synchronizers/Errors/Outlook/ObjectCannotBeDeletedHandler.cs index 08766a18..f31ba474 100644 --- a/Wino.Core/Synchronizers/Errors/Outlook/ObjectCannotBeDeletedHandler.cs +++ b/Wino.Core/Synchronizers/Errors/Outlook/ObjectCannotBeDeletedHandler.cs @@ -1,8 +1,8 @@ using System.Threading.Tasks; using Microsoft.Kiota.Abstractions; using Wino.Core.Domain.Interfaces; -using Wino.Core.Domain.Models.Errors; using Wino.Core.Domain.Models.Requests; +using Wino.Core.Domain.Models.Synchronization; using Wino.Core.Requests.Bundles; namespace Wino.Core.Synchronizers.Errors.Outlook; diff --git a/Wino.Core/Synchronizers/GmailSynchronizer.cs b/Wino.Core/Synchronizers/GmailSynchronizer.cs index 5de8df59..b390a9a6 100644 --- a/Wino.Core/Synchronizers/GmailSynchronizer.cs +++ b/Wino.Core/Synchronizers/GmailSynchronizer.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using System.Web; using CommunityToolkit.Mvvm.Messaging; using Google; -using Google.Apis.Calendar.v3; using Google.Apis.Calendar.v3.Data; using Google.Apis.Gmail.v1; using Google.Apis.Gmail.v1.Data; @@ -28,7 +27,6 @@ using Wino.Core.Domain.Enums; using Wino.Core.Domain.Exceptions; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.Accounts; -using Wino.Core.Domain.Models.Errors; using Wino.Core.Domain.Models.Folders; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; @@ -51,19 +49,22 @@ namespace Wino.Core.Synchronizers.Mail; public partial class GmailSynchronizerJsonContext : JsonSerializerContext; /// -/// Gmail synchronizer implementation with per-folder history ID synchronization. -/// +/// Gmail synchronizer implementation using Gmail History API for efficient incremental sync. +/// /// SYNCHRONIZATION STRATEGY: -/// - Uses Gmail History API for both initial and incremental sync -/// - Initial sync: Downloads top 1500 messages per folder with metadata only -/// - Incremental sync: Uses history ID to get only changes since last sync -/// - Messages are downloaded with metadata only (no MIME content during sync) -/// - MIME files are downloaded on-demand when user explicitly reads a message -/// +/// - Initial sync: Downloads up to 1500 messages PER FOLDER with metadata only. +/// Uses a global HashSet to track downloaded message IDs, avoiding duplicate downloads +/// when messages have multiple labels. Each folder gets its full quota of messages. +/// - Incremental sync: Uses ONLY History API to get changes since last sync. +/// No per-folder downloads during incremental sync - this is the proper Gmail sync approach. +/// - Messages are downloaded with metadata only during initial sync (no MIME content) +/// - New messages during incremental sync are downloaded with full MIME content +/// - MIME files for initial sync messages are downloaded on-demand when user reads a message +/// /// Key implementation details: -/// - SynchronizeFolderAsync: Main entry point for per-folder synchronization -/// - DownloadMessagesForFolderAsync: Downloads top 1500 messages for initial sync -/// - SynchronizeDeltaAsync: Processes incremental changes using history ID +/// - PerformInitialSyncAsync: Downloads messages per-folder with global deduplication +/// - SynchronizeDeltaAsync: Processes incremental changes using History API with pagination +/// - Handles 404/410 errors (history expired) by triggering full resync /// - CreateMinimalMailCopyAsync: Extracts MailCopy fields from Gmail Metadata format /// - DownloadMissingMimeMessageAsync: Downloads raw MIME only when explicitly requested /// @@ -153,15 +154,16 @@ public class GmailSynchronizer : WinoSynchronizer(); + var folderResults = new List(); - // Gmail must always synchronize folders before because it doesn't have a per-folder sync. - bool shouldSynchronizeFolders = true; - - if (shouldSynchronizeFolders) + try { + // Make sure that virtual archive folder exists before all. + if (!archiveFolderId.HasValue) + await InitializeArchiveFolderAsync().ConfigureAwait(false); + + // Gmail must always synchronize folders before because it doesn't have a per-folder sync. _logger.Information("Synchronizing folders for {Name}", Account.Name); UpdateSyncProgress(0, 0, "Synchronizing folders..."); @@ -173,202 +175,291 @@ public class GmailSynchronizer : WinoSynchronizer f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID)) + { + folderResults.Add(FolderSyncResult.Successful(folder.Id, folder.FolderName, 0)); + } + } + else + { + // INCREMENTAL SYNC: Use ONLY History API - no per-folder downloads. + // This is the proper Gmail sync strategy as recommended by Google. + UpdateSyncProgress(0, 0, "Synchronizing changes..."); + var deltaResult = await SynchronizeDeltaAsync(options, cancellationToken).ConfigureAwait(false); + downloadedMessageIds.AddRange(deltaResult.DownloadedMessageIds); + + // If history sync was reset due to expired history ID, we need to do initial sync + if (deltaResult.RequiresFullResync) + { + _logger.Warning("History ID expired. Performing full resync for {Name}", Account.Name); + downloadedMessageIds = await PerformInitialSyncAsync(cancellationToken).ConfigureAwait(false); + + // Update history ID after full resync + var profile = await _gmailService.Users.GetProfile("me").ExecuteAsync(cancellationToken); + if (profile.HistoryId.HasValue) + { + await UpdateAccountSyncIdentifierAsync(profile.HistoryId.Value).ConfigureAwait(false); + _logger.Information("Full resync completed. Set history ID to {HistoryId}", profile.HistoryId.Value); + } + } + + UpdateSyncProgress(0, 0, "Changes synchronized"); + + // Create folder results for incremental sync + var allFolders = await _gmailChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false); + foreach (var folder in allFolders.Where(f => f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID)) + { + folderResults.Add(FolderSyncResult.Successful(folder.Id, folder.FolderName, 0)); + } + } } - - // There is no specific folder synchronization in Gmail. - // Therefore we need to stop the synchronization at this point - // if type is only folder metadata sync. - - if (options.Type == MailSynchronizationType.FoldersOnly) return MailSynchronizationResult.Empty; - - cancellationToken.ThrowIfCancellationRequested(); - - bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier); - - _logger.Debug("Is initial synchronization: {IsInitialSync}", isInitialSync); - - var downloadedMessageIds = new List(); - - // Get all folders to synchronize - var synchronizationFolders = await _gmailChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false); - - _logger.Information("Synchronizing {Count} folders for {Name}", synchronizationFolders.Count, Account.Name); - - var totalFolders = synchronizationFolders.Count; - - for (int i = 0; i < totalFolders; i++) + catch (OperationCanceledException) { - var folder = synchronizationFolders[i]; - - // Update progress based on folder completion - UpdateSyncProgress(totalFolders, totalFolders - (i + 1), $"Syncing {folder.FolderName}..."); - - var folderDownloadedMessageIds = await SynchronizeFolderAsync(folder, cancellationToken).ConfigureAwait(false); - downloadedMessageIds.AddRange(folderDownloadedMessageIds); + _logger.Information("Synchronization was canceled for {Name}", Account.Name); + return MailSynchronizationResult.Canceled; } - - // Process incremental changes using history API if we have a history ID - if (!string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier)) + catch (Exception ex) { - UpdateSyncProgress(0, 0, "Synchronizing changes..."); - await SynchronizeDeltaAsync(options, cancellationToken).ConfigureAwait(false); - UpdateSyncProgress(0, 0, "Changes synchronized"); + _logger.Error(ex, "Synchronization failed for {Name}", Account.Name); + return MailSynchronizationResult.Failed(ex); } // Get all unread new downloaded items for notifications var unreadNewItems = await _gmailChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false); - return MailSynchronizationResult.Completed(unreadNewItems); + return MailSynchronizationResult.CompletedWithFolderResults(unreadNewItems, folderResults); } /// - /// Synchronizes a single folder by downloading top 1500 messages with metadata only. + /// Result of delta synchronization using History API. /// - private async Task> SynchronizeFolderAsync(MailItemFolder folder, CancellationToken cancellationToken) - { - var downloadedMessageIds = new List(); - - cancellationToken.ThrowIfCancellationRequested(); - - _logger.Debug("Synchronizing folder {FolderName} (label: {LabelId})", folder.FolderName, folder.RemoteFolderId); - - try - { - // Download top 1500 messages for this folder - await DownloadMessagesForFolderAsync(folder, downloadedMessageIds, cancellationToken).ConfigureAwait(false); - - if (downloadedMessageIds.Any()) - { - _logger.Information("Downloaded {Count} messages for folder {FolderName}", downloadedMessageIds.Count, folder.FolderName); - } - } - catch (Exception ex) - { - _logger.Error(ex, "Error synchronizing folder {FolderName}", folder.FolderName); - throw; - } - - return downloadedMessageIds; - } + private record DeltaSyncResult(List DownloadedMessageIds, bool RequiresFullResync); /// - /// Downloads top 1500 messages for a folder using Gmail API with metadata only. + /// Performs initial synchronization by downloading messages per-folder. + /// Each folder gets up to 1500 messages, but we track already downloaded message IDs globally + /// to avoid downloading the same message multiple times (Gmail messages can have multiple labels). /// - private async Task DownloadMessagesForFolderAsync(MailItemFolder folder, List downloadedMessageIds, CancellationToken cancellationToken) + private async Task> PerformInitialSyncAsync(CancellationToken cancellationToken) { - _logger.Debug("Downloading messages for folder {FolderName}", folder.FolderName); + // Track all downloaded message IDs globally to avoid duplicate downloads + var downloadedMessageIds = new HashSet(); + + _logger.Information("Performing initial sync for {Name} - downloading messages per folder", Account.Name); try { - var totalDownloaded = 0; - string pageToken = null; + // Get all folders to sync (exclude virtual ARCHIVE folder) + var folders = await _gmailChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false); + var syncableFolders = folders + .Where(f => f.IsSynchronizationEnabled && f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID) + .ToList(); - // Gmail API returns messages newest first by default - // We'll download up to 1500 messages per folder - var remainingToDownload = (int)InitialMessageDownloadCountPerFolder; + var totalFolders = syncableFolders.Count; + var totalMessagesDownloaded = 0; - do + for (int i = 0; i < totalFolders; i++) { + var folder = syncableFolders[i]; cancellationToken.ThrowIfCancellationRequested(); - var request = _gmailService.Users.Messages.List("me"); - request.LabelIds = new Google.Apis.Util.Repeatable(new[] { folder.RemoteFolderId }); - request.MaxResults = Math.Min(remainingToDownload, 500); // API max is 500 - request.PageToken = pageToken; + UpdateSyncProgress(totalFolders, totalFolders - (i + 1), $"Syncing {folder.FolderName}..."); - var response = await request.ExecuteAsync(cancellationToken); + _logger.Debug("Downloading messages for folder {FolderName} (label: {LabelId})", folder.FolderName, folder.RemoteFolderId); - if (response.Messages != null && response.Messages.Count > 0) + var folderDownloaded = 0; + string pageToken = null; + var remainingToDownload = (int)InitialMessageDownloadCountPerFolder; + + do { - var messageIds = response.Messages.Select(m => m.Id).ToList(); + cancellationToken.ThrowIfCancellationRequested(); - // Download metadata in batches - await DownloadMessagesInBatchAsync(messageIds, downloadRawMime: false, cancellationToken).ConfigureAwait(false); + var request = _gmailService.Users.Messages.List("me"); + request.LabelIds = new Google.Apis.Util.Repeatable(new[] { folder.RemoteFolderId }); + request.MaxResults = Math.Min(remainingToDownload, 500); // API max is 500 + request.PageToken = pageToken; - downloadedMessageIds.AddRange(messageIds); - totalDownloaded += messageIds.Count; - remainingToDownload -= messageIds.Count; + var response = await request.ExecuteAsync(cancellationToken); - _logger.Debug("Downloaded {Count} messages for folder {FolderName} (total: {Total})", messageIds.Count, folder.FolderName, totalDownloaded); + if (response.Messages != null && response.Messages.Count > 0) + { + // Filter out already downloaded messages to avoid duplicates + var newMessageIds = response.Messages + .Select(m => m.Id) + .Where(id => !downloadedMessageIds.Contains(id)) + .ToList(); - // Update progress - UpdateSyncProgress(0, 0, $"Downloaded {totalDownloaded} messages from {folder.FolderName}"); - } + if (newMessageIds.Count > 0) + { + // Download metadata in batches (no raw MIME during initial sync) + await DownloadMessagesInBatchAsync(newMessageIds, downloadRawMime: false, cancellationToken).ConfigureAwait(false); - pageToken = response.NextPageToken; + foreach (var id in newMessageIds) + { + downloadedMessageIds.Add(id); + } - // Stop if we've downloaded enough messages or no more pages - if (remainingToDownload <= 0 || string.IsNullOrEmpty(pageToken)) - break; + folderDownloaded += newMessageIds.Count; + totalMessagesDownloaded += newMessageIds.Count; + } - } while (!string.IsNullOrEmpty(pageToken)); + // Count all messages (including duplicates) toward the folder limit + remainingToDownload -= response.Messages.Count; - // Store history ID for future incremental syncs - var profile = await _gmailService.Users.GetProfile("me").ExecuteAsync(cancellationToken); - Account.SynchronizationDeltaIdentifier = profile.HistoryId.ToString(); - await _gmailChangeProcessor.UpdateAccountAsync(Account).ConfigureAwait(false); + _logger.Debug("Folder {FolderName}: Downloaded {New} new messages ({Total} total in folder)", + folder.FolderName, newMessageIds.Count, folderDownloaded); + } - _logger.Information("Completed downloading {Count} messages for folder {FolderName}", totalDownloaded, folder.FolderName); + pageToken = response.NextPageToken; + + // Stop if we've processed enough messages for this folder or no more pages + if (remainingToDownload <= 0 || string.IsNullOrEmpty(pageToken)) + break; + + } while (!string.IsNullOrEmpty(pageToken)); + + _logger.Information("Folder {FolderName}: Downloaded {Count} messages", folder.FolderName, folderDownloaded); + UpdateSyncProgress(0, 0, $"Downloaded {totalMessagesDownloaded} messages"); + } + + _logger.Information("Initial sync completed. Downloaded {Count} unique messages for {Name}", downloadedMessageIds.Count, Account.Name); } catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests) { - _logger.Warning("Rate limit exceeded while downloading messages for folder {FolderName}. Retrying after delay.", folder.FolderName); + _logger.Warning("Rate limit exceeded during initial sync. Retrying after delay."); await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); throw; } catch (Exception ex) { - _logger.Error(ex, "Error downloading messages for folder {FolderName}", folder.FolderName); + _logger.Error(ex, "Error during initial sync for {Name}", Account.Name); throw; } + + return downloadedMessageIds.ToList(); } - private async Task SynchronizeDeltaAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default) + /// + /// Performs incremental synchronization using Gmail History API. + /// This is the recommended approach for Gmail sync after initial sync is complete. + /// Returns a result indicating downloaded messages and whether a full resync is needed. + /// + private async Task SynchronizeDeltaAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default) { + var downloadedMessageIds = new List(); + try { - var historyRequest = _gmailService.Users.History.List("me"); - historyRequest.StartHistoryId = ulong.Parse(Account.SynchronizationDeltaIdentifier!); + string pageToken = null; - var historyResponse = await historyRequest.ExecuteAsync(); - - if (historyResponse.History != null) + do { - var addedMessageIds = new List(); + cancellationToken.ThrowIfCancellationRequested(); - // Collect all added messages first - foreach (var historyRecord in historyResponse.History) + var historyRequest = _gmailService.Users.History.List("me"); + historyRequest.StartHistoryId = ulong.Parse(Account.SynchronizationDeltaIdentifier!); + + if (!string.IsNullOrEmpty(pageToken)) + historyRequest.PageToken = pageToken; + + var historyResponse = await historyRequest.ExecuteAsync(cancellationToken); + + if (historyResponse.History != null) { - if (historyRecord.MessagesAdded != null) + var addedMessageIds = new List(); + + // Collect all added messages first + foreach (var historyRecord in historyResponse.History) { - addedMessageIds.AddRange(historyRecord.MessagesAdded.Select(ma => ma.Message.Id)); + if (historyRecord.MessagesAdded != null) + { + addedMessageIds.AddRange(historyRecord.MessagesAdded.Select(ma => ma.Message.Id)); + } } - } - // Process added messages in batches if any - // During delta sync, download with Raw format to get MIME content - if (addedMessageIds.Count != 0) - { - await DownloadMessagesInBatchAsync(addedMessageIds, downloadRawMime: true, cancellationToken).ConfigureAwait(false); - } + // Process added messages in batches if any + // During delta sync, download with Raw format to get MIME content for new messages + if (addedMessageIds.Count != 0) + { + // Deduplicate message IDs + var uniqueAddedIds = addedMessageIds.Distinct().ToList(); + await DownloadMessagesInBatchAsync(uniqueAddedIds, downloadRawMime: true, cancellationToken).ConfigureAwait(false); + downloadedMessageIds.AddRange(uniqueAddedIds); + } - // Process other history changes - foreach (var historyRecord in historyResponse.History) - { + // Process other history changes (label changes, deletions) await ProcessHistoryChangesAsync(historyResponse).ConfigureAwait(false); } - } - } - catch (Exception) - { + // CRITICAL: Update the history ID to the latest one after processing all changes + // History IDs are always incremental, so the response contains the latest history ID + if (historyResponse.HistoryId.HasValue) + { + await UpdateAccountSyncIdentifierAsync(historyResponse.HistoryId.Value).ConfigureAwait(false); + _logger.Debug("Updated history ID to {HistoryId} after delta sync", historyResponse.HistoryId.Value); + } + + pageToken = historyResponse.NextPageToken; + + } while (!string.IsNullOrEmpty(pageToken)); + + _logger.Information("Delta sync completed. Downloaded {Count} new messages for {Name}", downloadedMessageIds.Count, Account.Name); + + return new DeltaSyncResult(downloadedMessageIds, RequiresFullResync: false); + } + catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound || + (int)ex.HttpStatusCode == 410) // Gone - history expired + { + // History ID is no longer valid (expired or not found) + // This happens when: + // 1. The history ID is too old (Gmail keeps history for ~30 days) + // 2. The account was reset or history was cleared + // Reset the sync identifier and signal that a full resync is needed + _logger.Warning("History ID {HistoryId} expired or not found for {Name}. Full resync required. Error: {Error}", + Account.SynchronizationDeltaIdentifier, Account.Name, ex.Message); + + // Clear the sync identifier to trigger initial sync + Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor + .UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, null) + .ConfigureAwait(false); + + return new DeltaSyncResult(downloadedMessageIds, RequiresFullResync: true); + } + catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + _logger.Warning("Rate limit exceeded during delta sync for {Name}. Retrying after delay.", Account.Name); + await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); throw; } } @@ -802,6 +893,13 @@ public class GmailSynchronizer : WinoSynchronizer h.Name.Equals("Content-Type", StringComparison.OrdinalIgnoreCase))?.Value; - + if (!string.IsNullOrEmpty(contentTypeHeader)) { // Check if it's a calendar message (text/calendar or multipart with calendar) @@ -1623,11 +1724,11 @@ public class GmailSynchronizer : WinoSynchronizer MailItemType.CalendarInvitation, @@ -1636,7 +1737,7 @@ public class GmailSynchronizer : WinoSynchronizer MailItemType.Mail }; } - + // If no method specified, assume it's an invitation return MailItemType.CalendarInvitation; } @@ -1683,11 +1784,11 @@ public class GmailSynchronizer : WinoSynchronizer(); - // Create MailCopy from metadata only - NO MIME download - var mailCopy = await CreateMinimalMailCopyAsync(message, assignedFolder, cancellationToken); + // Create base MailCopy from metadata only - NO MIME download + var baseMailCopy = await CreateMinimalMailCopyAsync(message, assignedFolder, cancellationToken); // Check for local draft mapping using X-Wino-Draft-Id header from metadata - if (mailCopy.IsDraft) + if (baseMailCopy.IsDraft) { var draftIdHeader = message.Payload?.Headers?.FirstOrDefault(h => h.Name.Equals(Domain.Constants.WinoLocalDraftHeader, StringComparison.OrdinalIgnoreCase))?.Value; @@ -1696,7 +1797,7 @@ public class GmailSynchronizer : WinoSynchronizer(patchRequest, request)]; @@ -1861,8 +1982,8 @@ public class GmailSynchronizer : WinoSynchronizer(patchRequest, request)]; @@ -1898,8 +2019,8 @@ public class GmailSynchronizer : WinoSynchronizer(patchRequest, request)]; @@ -1979,8 +2100,8 @@ public class GmailSynchronizer : WinoSynchronizer 0) - ? Google.Apis.Calendar.v3.EventsResource.UpdateRequest.SendUpdatesEnum.All + updateRequest.SendUpdates = (attendees != null && attendees.Count > 0) + ? Google.Apis.Calendar.v3.EventsResource.UpdateRequest.SendUpdatesEnum.All : Google.Apis.Calendar.v3.EventsResource.UpdateRequest.SendUpdatesEnum.None; return [new HttpRequestBundle(updateRequest, request)]; diff --git a/Wino.Core/Synchronizers/ImapSync/UnifiedImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSync/UnifiedImapSynchronizer.cs new file mode 100644 index 00000000..e1d32b4c --- /dev/null +++ b/Wino.Core/Synchronizers/ImapSync/UnifiedImapSynchronizer.cs @@ -0,0 +1,473 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MailKit; +using MailKit.Net.Imap; +using MailKit.Search; +using MoreLinq; +using Serilog; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.MailItem; +using Wino.Core.Domain.Models.Synchronization; +using Wino.Core.Integration; +using Wino.Services.Extensions; +using IMailService = Wino.Core.Domain.Interfaces.IMailService; + +namespace Wino.Core.Synchronizers.ImapSync; + +/// +/// Unified IMAP synchronization strategy that automatically selects the best available method: +/// 1. QRESYNC (RFC 5162) - Best: supports quick resync with vanished messages +/// 2. CONDSTORE (RFC 4551) - Good: supports mod-seq based change tracking +/// 3. UID-based - Fallback: basic UID comparison +/// +/// This consolidates the previous QResyncSynchronizer, CondstoreSynchronizer, and UidBasedSynchronizer +/// into a single, enterprise-grade implementation with proper error handling and partial failure support. +/// +public class UnifiedImapSynchronizer +{ + private readonly ILogger _logger = Log.ForContext(); + private readonly IFolderService _folderService; + private readonly IMailService _mailService; + private readonly IImapSynchronizerErrorHandlerFactory _errorHandlerFactory; + + // Minimum summary items to Fetch for mail synchronization from IMAP. + private readonly MessageSummaryItems MailSynchronizationFlags = + MessageSummaryItems.Flags | + MessageSummaryItems.UniqueId | + MessageSummaryItems.ThreadId | + MessageSummaryItems.EmailId | + MessageSummaryItems.Headers | + MessageSummaryItems.PreviewText | + MessageSummaryItems.GMailThreadId | + MessageSummaryItems.References | + MessageSummaryItems.ModSeq; + + public UnifiedImapSynchronizer( + IFolderService folderService, + IMailService mailService, + IImapSynchronizerErrorHandlerFactory errorHandlerFactory) + { + _folderService = folderService; + _mailService = mailService; + _errorHandlerFactory = errorHandlerFactory; + } + + /// + /// Determines the best synchronization strategy based on server capabilities. + /// + public ImapSyncStrategy DetermineSyncStrategy(IImapClient client) + { + if (client is WinoImapClient winoClient && + client.Capabilities.HasFlag(ImapCapabilities.QuickResync) && + winoClient.IsQResyncEnabled) + { + return ImapSyncStrategy.QResync; + } + + if (client.Capabilities.HasFlag(ImapCapabilities.CondStore)) + { + return ImapSyncStrategy.Condstore; + } + + return ImapSyncStrategy.UidBased; + } + + /// + /// Main synchronization entry point. Automatically selects the best strategy. + /// + public async Task SynchronizeFolderAsync( + IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken = default) + { + var strategy = DetermineSyncStrategy(client); + _logger.Debug("Using {Strategy} sync strategy for folder {FolderName}", strategy, folder.FolderName); + + try + { + var downloadedIds = strategy switch + { + ImapSyncStrategy.QResync => await SynchronizeWithQResyncAsync(client, folder, synchronizer, cancellationToken), + ImapSyncStrategy.Condstore => await SynchronizeWithCondstoreAsync(client, folder, synchronizer, cancellationToken), + _ => await SynchronizeWithUidBasedAsync(client, folder, synchronizer, cancellationToken) + }; + + return FolderSyncResult.Successful(folder.Id, folder.FolderName, downloadedIds.Count); + } + catch (FolderNotFoundException) + { + _logger.Warning("Folder {FolderName} not found on server, deleting locally", folder.FolderName); + await _folderService.DeleteFolderAsync(folder.MailAccountId, folder.RemoteFolderId).ConfigureAwait(false); + + return FolderSyncResult.Skipped(folder.Id, folder.FolderName, "Folder not found on server"); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + var errorContext = new SynchronizerErrorContext + { + ErrorMessage = ex.Message, + Exception = ex, + FolderId = folder.Id, + FolderName = folder.FolderName, + OperationType = "ImapFolderSync" + }; + + var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false); + + if (errorContext.CanContinueSync) + { + _logger.Warning(ex, "Folder {FolderName} sync failed with recoverable error", folder.FolderName); + return FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext); + } + + _logger.Error(ex, "Folder {FolderName} sync failed with fatal error", folder.FolderName); + throw; + } + } + + #region QRESYNC Strategy + + private async Task> SynchronizeWithQResyncAsync( + IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken) + { + var downloadedMessageIds = new List(); + + if (client is not WinoImapClient winoClient) + throw new InvalidOperationException("QRESYNC requires WinoImapClient"); + + IMailFolder remoteFolder = null; + + try + { + remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); + + var localHighestModSeq = (ulong)Math.Max(folder.HighestModeSeq, 1); + var allUids = await _folderService.GetKnownUidsForFolderAsync(folder.Id); + var allUniqueIds = allUids.Select(a => new UniqueId(a)).ToList(); + + // Subscribe to events before opening + remoteFolder.MessagesVanished += (s, e) => HandleMessagesVanished(folder, e.UniqueIds); + remoteFolder.MessageFlagsChanged += (s, e) => HandleMessageFlagsChanged(folder, e.UniqueId, e.Flags); + + // Open with QRESYNC parameters + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, allUniqueIds, cancellationToken).ConfigureAwait(false); + + // Get changed UIDs + var changedUids = await remoteFolder.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken).ConfigureAwait(false); + + downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false); + + // Update folder tracking + folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq); + folder.UidValidity = remoteFolder.UidValidity; + + // Handle deletions + await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); + + await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false); + } + finally + { + if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) + { + await remoteFolder.CloseAsync().ConfigureAwait(false); + } + } + + return downloadedMessageIds; + } + + #endregion + + #region CONDSTORE Strategy + + private async Task> SynchronizeWithCondstoreAsync( + IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken) + { + var downloadedMessageIds = new List(); + IMailFolder remoteFolder = null; + + try + { + remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); + + var localHighestModSeq = (ulong)folder.HighestModeSeq; + bool isInitialSync = localHighestModSeq == 0; + + if (remoteFolder.HighestModSeq > localHighestModSeq || isInitialSync) + { + IList changedUids; + + // Use SORT if available for better ordering + if (client.Capabilities.HasFlag(ImapCapabilities.Sort)) + { + changedUids = await remoteFolder.SortAsync( + SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), + [OrderBy.ReverseDate], + cancellationToken).ConfigureAwait(false); + } + else + { + changedUids = await remoteFolder.SearchAsync( + SearchQuery.ChangedSince(Math.Max(localHighestModSeq, 1)), + cancellationToken).ConfigureAwait(false); + } + + // For initial sync, limit the number of messages + if (isInitialSync) + { + changedUids = changedUids + .OrderByDescending(a => a.Id) + .Take((int)synchronizer.InitialMessageDownloadCountPerFolder) + .ToList(); + } + + downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, changedUids, cancellationToken).ConfigureAwait(false); + + folder.HighestModeSeq = unchecked((long)remoteFolder.HighestModSeq); + await _folderService.UpdateFolderAsync(folder).ConfigureAwait(false); + } + + await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); + } + finally + { + if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) + { + await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + return downloadedMessageIds; + } + + #endregion + + #region UID-Based Strategy (Fallback) + + private async Task> SynchronizeWithUidBasedAsync( + IImapClient client, + MailItemFolder folder, + IImapSynchronizer synchronizer, + CancellationToken cancellationToken) + { + var downloadedMessageIds = new List(); + IMailFolder remoteFolder = null; + + try + { + remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId, cancellationToken).ConfigureAwait(false); + await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false); + + // Get all remote UIDs and take the most recent ones + var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false); + var limitedUids = remoteUids + .OrderByDescending(a => a.Id) + .Take((int)synchronizer.InitialMessageDownloadCountPerFolder) + .ToList(); + + downloadedMessageIds = await ProcessChangedUidsAsync(synchronizer, remoteFolder, folder, limitedUids, cancellationToken).ConfigureAwait(false); + + await HandleDeletedMessagesAsync(folder, remoteFolder, cancellationToken).ConfigureAwait(false); + } + finally + { + if (remoteFolder?.IsOpen == true && !cancellationToken.IsCancellationRequested) + { + await remoteFolder.CloseAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + return downloadedMessageIds; + } + + #endregion + + #region Shared Processing Methods + + private async Task> ProcessChangedUidsAsync( + IImapSynchronizer synchronizer, + IMailFolder remoteFolder, + MailItemFolder localFolder, + IList changedUids, + CancellationToken cancellationToken) + { + var downloadedMessageIds = new List(); + + if (changedUids == null || changedUids.Count == 0) + return downloadedMessageIds; + + // Get existing mails to determine what's new vs. updated + var existingMails = await _mailService.GetExistingMailsAsync(localFolder.Id, changedUids).ConfigureAwait(false); + var existingMailUids = existingMails.Select(m => MailkitClientExtensions.ResolveUidStruct(m.Id)).ToArray(); + + var newMessageUids = changedUids.Except(existingMailUids).ToList(); + + // Update flags for existing mails + if (existingMailUids.Any()) + { + var existingFlagData = await remoteFolder.FetchAsync(existingMailUids, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId, cancellationToken).ConfigureAwait(false); + + foreach (var update in existingFlagData) + { + if (update.UniqueId == UniqueId.Invalid || update.Flags == null) continue; + + var existingMail = existingMails.FirstOrDefault(m => MailkitClientExtensions.ResolveUidStruct(m.Id).Id == update.UniqueId.Id); + if (existingMail != null) + { + await UpdateMailFlagsAsync(existingMail, update.Flags.Value).ConfigureAwait(false); + } + } + } + + // Download new messages in batches + var batches = newMessageUids.Batch(50); + foreach (var batch in batches) + { + cancellationToken.ThrowIfCancellationRequested(); + + var batchList = batch.ToList(); + downloadedMessageIds.AddRange(batchList.Select(uid => MailkitClientExtensions.CreateUid(localFolder.Id, uid.Id))); + + await DownloadMessagesAsync(synchronizer, remoteFolder, localFolder, new UniqueIdSet(batchList, SortOrder.Ascending), cancellationToken).ConfigureAwait(false); + } + + return downloadedMessageIds; + } + + private async Task DownloadMessagesAsync( + IImapSynchronizer synchronizer, + IMailFolder folder, + MailItemFolder localFolder, + UniqueIdSet uniqueIdSet, + CancellationToken cancellationToken) + { + var summaries = await folder.FetchAsync(uniqueIdSet, MailSynchronizationFlags, cancellationToken).ConfigureAwait(false); + + foreach (var summary in summaries) + { + try + { + var mimeMessage = await folder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false); + var creationPackage = new ImapMessageCreationPackage(summary, mimeMessage); + var mailPackages = await synchronizer.CreateNewMailPackagesAsync(creationPackage, localFolder, cancellationToken).ConfigureAwait(false); + + if (mailPackages != null) + { + foreach (var package in mailPackages) + { + if (package != null) + { + await _mailService.CreateMailAsync(localFolder.MailAccountId, package).ConfigureAwait(false); + } + } + } + } + catch (Exception ex) + { + _logger.Warning(ex, "Failed to download message {UniqueId} in folder {FolderName}", summary.UniqueId, localFolder.FolderName); + // Continue with other messages + } + } + } + + private async Task HandleDeletedMessagesAsync(MailItemFolder localFolder, IMailFolder remoteFolder, CancellationToken cancellationToken) + { + var allLocalUids = (await _folderService.GetKnownUidsForFolderAsync(localFolder.Id)).Select(a => new UniqueId(a)).ToList(); + + if (allLocalUids.Count == 0) return; + + var remoteAllUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false); + var deletedUids = allLocalUids.Except(remoteAllUids).ToList(); + + foreach (var deletedUid in deletedUids) + { + var localMailCopyId = MailkitClientExtensions.CreateUid(localFolder.Id, deletedUid.Id); + await _mailService.DeleteMailAsync(localFolder.MailAccountId, localMailCopyId).ConfigureAwait(false); + } + } + + private async Task UpdateMailFlagsAsync(MailCopy mailCopy, MessageFlags flags) + { + var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); + var isRead = MailkitClientExtensions.GetIsRead(flags); + + if (isFlagged != mailCopy.IsFlagged) + { + await _mailService.ChangeFlagStatusAsync(mailCopy.Id, isFlagged).ConfigureAwait(false); + } + + if (isRead != mailCopy.IsRead) + { + await _mailService.ChangeReadStatusAsync(mailCopy.Id, isRead).ConfigureAwait(false); + } + } + + private void HandleMessagesVanished(MailItemFolder folder, IList uniqueIds) + { + // Fire and forget - these are event handlers + _ = Task.Run(async () => + { + foreach (var uniqueId in uniqueIds) + { + var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id); + await _mailService.DeleteMailAsync(folder.MailAccountId, localMailCopyId).ConfigureAwait(false); + } + }); + } + + private void HandleMessageFlagsChanged(MailItemFolder folder, UniqueId? uniqueId, MessageFlags flags) + { + if (uniqueId == null) return; + + _ = Task.Run(async () => + { + var localMailCopyId = MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Value.Id); + var isFlagged = MailkitClientExtensions.GetIsFlagged(flags); + var isRead = MailkitClientExtensions.GetIsRead(flags); + + await _mailService.ChangeReadStatusAsync(localMailCopyId, isRead).ConfigureAwait(false); + await _mailService.ChangeFlagStatusAsync(localMailCopyId, isFlagged).ConfigureAwait(false); + }); + } + + #endregion +} + +/// +/// IMAP synchronization strategy enumeration. +/// +public enum ImapSyncStrategy +{ + /// + /// RFC 5162 Quick Resync - supports vanished messages and efficient delta sync. + /// + QResync, + + /// + /// RFC 4551 Conditional Store - supports mod-seq based change tracking. + /// + Condstore, + + /// + /// Basic UID-based synchronization - fallback for servers without advanced features. + /// + UidBased +} diff --git a/Wino.Core/Synchronizers/ImapSynchronizer.cs b/Wino.Core/Synchronizers/ImapSynchronizer.cs index 6d9198e5..2fc17dbf 100644 --- a/Wino.Core/Synchronizers/ImapSynchronizer.cs +++ b/Wino.Core/Synchronizers/ImapSynchronizer.cs @@ -26,6 +26,7 @@ using Wino.Core.Integration.Processors; using Wino.Core.Requests.Bundles; using Wino.Core.Requests.Folder; using Wino.Core.Requests.Mail; +using Wino.Core.Synchronizers.ImapSync; using Wino.Messaging.Server; using Wino.Messaging.UI; using Wino.Services.Extensions; @@ -52,16 +53,22 @@ public class ImapSynchronizer : WinoSynchronizer SynchronizeMailsInternalAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default) { var downloadedMessageIds = new List(); + var folderResults = new List(); _logger.Information("Internal synchronization started for {Name}", Account.Name); _logger.Information("Options: {Options}", options); - // Set indeterminate progress initially - UpdateSyncProgress(0, 0, "Synchronizing..."); - - bool shouldDoFolderSync = options.Type == MailSynchronizationType.FullFolders || options.Type == MailSynchronizationType.FoldersOnly; - - if (shouldDoFolderSync) + try { - await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false); - } + // Set indeterminate progress initially + UpdateSyncProgress(0, 0, "Synchronizing..."); - if (options.Type != MailSynchronizationType.FoldersOnly) - { - var synchronizationFolders = await _imapChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false); + bool shouldDoFolderSync = options.Type == MailSynchronizationType.FullFolders || options.Type == MailSynchronizationType.FoldersOnly; - var totalFolders = synchronizationFolders.Count; - - for (int i = 0; i < totalFolders; i++) + if (shouldDoFolderSync) { - var folder = synchronizationFolders[i]; + await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false); + } - // Update progress based on folder completion - UpdateSyncProgress(totalFolders, totalFolders - (i + 1), $"Syncing {folder.FolderName}..."); + if (options.Type != MailSynchronizationType.FoldersOnly) + { + var synchronizationFolders = await _imapChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false); - var folderDownloadedMessageIds = await SynchronizeFolderInternalAsync(folder, cancellationToken).ConfigureAwait(false); + var totalFolders = synchronizationFolders.Count; - if (cancellationToken.IsCancellationRequested) return MailSynchronizationResult.Canceled; - - if (folderDownloadedMessageIds != null) + for (int i = 0; i < totalFolders; i++) { - downloadedMessageIds.AddRange(folderDownloadedMessageIds); + var folder = synchronizationFolders[i]; + + // Update progress based on folder completion + UpdateSyncProgress(totalFolders, totalFolders - (i + 1), $"Syncing {folder.FolderName}..."); + + try + { + // Use the unified synchronizer for folder sync + IImapClient client = null; + + try + { + client = await _clientPool.GetClientAsync().ConfigureAwait(false); + var folderResult = await _unifiedSynchronizer.SynchronizeFolderAsync(client, folder, this, cancellationToken).ConfigureAwait(false); + folderResults.Add(folderResult); + + if (folderResult.Success && folderResult.DownloadedCount > 0) + { + // Get the downloaded message IDs for this folder + var folderDownloadedIds = await GetDownloadedIdsForFolderAsync(folder, folderResult.DownloadedCount).ConfigureAwait(false); + downloadedMessageIds.AddRange(folderDownloadedIds); + } + } + finally + { + if (client != null) + { + _clientPool.Release(client); + } + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + var errorContext = new SynchronizerErrorContext + { + Account = Account, + ErrorMessage = ex.Message, + Exception = ex, + FolderId = folder.Id, + FolderName = folder.FolderName, + OperationType = "ImapFolderSync" + }; + + var handled = await _errorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false); + + if (errorContext.CanContinueSync) + { + _logger.Warning(ex, "Folder {FolderName} sync failed, continuing with other folders", folder.FolderName); + folderResults.Add(FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext)); + } + else + { + _logger.Error(ex, "Folder {FolderName} sync failed with fatal error", folder.FolderName); + folderResults.Add(FolderSyncResult.Failed(folder.Id, folder.FolderName, errorContext)); + throw; + } + } + + if (cancellationToken.IsCancellationRequested) return MailSynchronizationResult.Canceled; } } } - - // Reset progress - ResetSyncProgress(); + catch (OperationCanceledException) + { + _logger.Information("Synchronization was canceled for {Name}", Account.Name); + return MailSynchronizationResult.Canceled; + } + catch (Exception ex) + { + _logger.Error(ex, "Synchronization failed for {Name}", Account.Name); + return MailSynchronizationResult.Failed(ex); + } + finally + { + // Reset progress + ResetSyncProgress(); + } // Get all unread new downloaded items and return in the result. // This is primarily used in notifications. var unreadNewItems = await _imapChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false); - return MailSynchronizationResult.Completed(unreadNewItems); + return MailSynchronizationResult.CompletedWithFolderResults(unreadNewItems, folderResults); + } + + /// + /// Gets the most recent downloaded message IDs for a folder. + /// Used for notification purposes after sync completes. + /// + private async Task> GetDownloadedIdsForFolderAsync(MailItemFolder folder, int count) + { + // Get the most recent mail IDs from the folder + var recentMails = await _imapChangeProcessor.GetRecentMailIdsForFolderAsync(folder.Id, count).ConfigureAwait(false); + return recentMails?.ToList() ?? new List(); } public override async Task ExecuteNativeRequestsAsync(List> batchedRequests, CancellationToken cancellationToken = default) diff --git a/Wino.Core/Synchronizers/OutlookSynchronizer.cs b/Wino.Core/Synchronizers/OutlookSynchronizer.cs index 7934969d..1fac6c5a 100644 --- a/Wino.Core/Synchronizers/OutlookSynchronizer.cs +++ b/Wino.Core/Synchronizers/OutlookSynchronizer.cs @@ -30,7 +30,6 @@ using Wino.Core.Domain.Enums; using Wino.Core.Domain.Exceptions; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.Accounts; -using Wino.Core.Domain.Models.Errors; using Wino.Core.Domain.Models.Folders; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; @@ -141,6 +140,7 @@ public class OutlookSynchronizer : WinoSynchronizer SynchronizeMailsInternalAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default) { var downloadedMessageIds = new List(); + var folderResults = new List(); _logger.Information("Internal synchronization started for {Name}", Account.Name); _logger.Information("Options: {Options}", options); @@ -169,17 +169,77 @@ public class OutlookSynchronizer : WinoSynchronizer + + + + \ No newline at end of file diff --git a/Wino.Mail.ViewModels/MailListPageViewModel.cs b/Wino.Mail.ViewModels/MailListPageViewModel.cs index 79399447..d49d3a56 100644 --- a/Wino.Mail.ViewModels/MailListPageViewModel.cs +++ b/Wino.Mail.ViewModels/MailListPageViewModel.cs @@ -670,7 +670,7 @@ public partial class MailListPageViewModel : MailBaseViewModel, await MailCollection.UpdateMailCopy(updatedMail, source); - await ExecuteUIThread(() => { SetupTopBarActions(); }); + // await ExecuteUIThread(() => { SetupTopBarActions(); }); } protected override async void OnMailRemoved(MailCopy removedMail) diff --git a/Wino.Services/AccountService.cs b/Wino.Services/AccountService.cs index e2f4ab14..1a66c6e5 100644 --- a/Wino.Services/AccountService.cs +++ b/Wino.Services/AccountService.cs @@ -599,4 +599,24 @@ public class AccountService : BaseDatabaseService, IAccountService return account?.Preferences?.IsNotificationsEnabled ?? false; } + + public async Task UpdateLastFolderStructureSyncDateAsync(Guid accountId) + { + var account = await GetAccountAsync(accountId); + if (account == null) return; + + account.LastFolderStructureSyncDate = DateTime.UtcNow; + await Connection.UpdateAsync(account, typeof(MailAccount)).ConfigureAwait(false); + } + + public async Task ShouldSyncFolderStructureAsync(Guid accountId, TimeSpan syncInterval) + { + var account = await GetAccountAsync(accountId); + if (account == null) return true; + + if (!account.LastFolderStructureSyncDate.HasValue) + return true; + + return DateTime.UtcNow - account.LastFolderStructureSyncDate.Value > syncInterval; + } } diff --git a/Wino.Services/MailService.cs b/Wino.Services/MailService.cs index e338f680..5b248a8a 100644 --- a/Wino.Services/MailService.cs +++ b/Wino.Services/MailService.cs @@ -1124,6 +1124,18 @@ public class MailService : BaseDatabaseService, IMailService return new GmailArchiveComparisonResult(addedMails, removedMails); } + public async Task> GetRecentMailIdsForFolderAsync(Guid folderId, int count) + { + var recentMails = await Connection.Table() + .Where(a => a.FolderId == folderId) + .OrderByDescending(a => a.CreationDate) + .Take(count) + .ToListAsync() + .ConfigureAwait(false); + + return recentMails.Select(m => m.Id); + } + public async Task> GetMailItemsAsync(IEnumerable mailCopyIds) { if (!mailCopyIds.Any()) return [];