Refactored all synchronizers to deal with some of the chronic issues.
This commit is contained in:
@@ -1,11 +1,22 @@
|
||||
using System.Threading.Tasks;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Errors;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Synchronizers.Errors.Gmail;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for handling Gmail synchronizer errors.
|
||||
/// Registers and routes errors to appropriate handlers.
|
||||
/// </summary>
|
||||
public class GmailSynchronizerErrorHandlingFactory : SynchronizerErrorHandlingFactory, IGmailSynchronizerErrorHandlerFactory
|
||||
{
|
||||
public bool CanHandle(SynchronizerErrorContext error) => CanHandle(error);
|
||||
|
||||
public Task HandleAsync(SynchronizerErrorContext error) => HandleErrorAsync(error);
|
||||
public GmailSynchronizerErrorHandlingFactory(
|
||||
GmailQuotaExceededHandler quotaExceededHandler,
|
||||
GmailRateLimitHandler rateLimitHandler,
|
||||
GmailHistoryExpiredHandler historyExpiredHandler)
|
||||
{
|
||||
// Order matters - more specific handlers should be registered first
|
||||
RegisterHandler(quotaExceededHandler);
|
||||
RegisterHandler(historyExpiredHandler);
|
||||
RegisterHandler(rateLimitHandler); // Most generic rate limit handler last
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Synchronizers.Errors.Imap;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Factory for handling IMAP synchronizer errors.
|
||||
/// Registers and routes errors to appropriate handlers.
|
||||
/// </summary>
|
||||
public class ImapSynchronizerErrorHandlingFactory : SynchronizerErrorHandlingFactory, IImapSynchronizerErrorHandlerFactory
|
||||
{
|
||||
public ImapSynchronizerErrorHandlingFactory(
|
||||
ImapConnectionLostHandler connectionLostHandler,
|
||||
ImapAuthenticationFailedHandler authFailedHandler,
|
||||
ImapFolderNotFoundHandler folderNotFoundHandler,
|
||||
ImapProtocolErrorHandler protocolErrorHandler)
|
||||
{
|
||||
// Order matters - more specific handlers should be registered first
|
||||
RegisterHandler(authFailedHandler);
|
||||
RegisterHandler(folderNotFoundHandler);
|
||||
RegisterHandler(connectionLostHandler);
|
||||
RegisterHandler(protocolErrorHandler); // Most generic, registered last
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
using System.Threading.Tasks;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Errors;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
using Wino.Core.Synchronizers.Errors.Outlook;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Retry;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Executes operations with automatic retry and error handling support.
|
||||
/// Implements exponential backoff with jitter.
|
||||
/// </summary>
|
||||
public class RetryExecutor : IRetryExecutor
|
||||
{
|
||||
private readonly ILogger _logger = Log.ForContext<RetryExecutor>();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task<T> ExecuteWithRetryAsync<T>(
|
||||
Func<CancellationToken, Task<T>> operation,
|
||||
RetryPolicy policy,
|
||||
Func<Exception, SynchronizerErrorContext> errorContextFactory,
|
||||
ISynchronizerErrorHandlerFactory errorHandler = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(operation);
|
||||
ArgumentNullException.ThrowIfNull(policy);
|
||||
ArgumentNullException.ThrowIfNull(errorContextFactory);
|
||||
|
||||
int attempt = 0;
|
||||
Exception lastException = null;
|
||||
|
||||
while (attempt <= policy.MaxRetries)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
return await operation(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw; // Don't retry on cancellation
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
lastException = ex;
|
||||
attempt++;
|
||||
|
||||
var errorContext = errorContextFactory(ex);
|
||||
errorContext.RetryCount = attempt;
|
||||
errorContext.MaxRetries = policy.MaxRetries;
|
||||
|
||||
// Let the error handler process the error first
|
||||
if (errorHandler != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
var handled = await errorHandler.HandleErrorAsync(errorContext).ConfigureAwait(false);
|
||||
if (handled)
|
||||
{
|
||||
_logger.Debug("Error handled by error handler, severity: {Severity}", errorContext.Severity);
|
||||
}
|
||||
}
|
||||
catch (Exception handlerEx)
|
||||
{
|
||||
_logger.Warning(handlerEx, "Error handler threw an exception");
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we should retry based on error severity
|
||||
if (errorContext.Severity == SynchronizerErrorSeverity.Fatal ||
|
||||
errorContext.Severity == SynchronizerErrorSeverity.AuthRequired)
|
||||
{
|
||||
_logger.Warning(ex, "Non-retryable error (severity: {Severity}), failing immediately", errorContext.Severity);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (errorContext.Severity == SynchronizerErrorSeverity.Recoverable)
|
||||
{
|
||||
_logger.Debug(ex, "Recoverable error, not retrying but allowing continuation");
|
||||
throw;
|
||||
}
|
||||
|
||||
// Transient error - check if we have retries left
|
||||
if (attempt > policy.MaxRetries)
|
||||
{
|
||||
_logger.Warning(ex, "All {MaxRetries} retries exhausted", policy.MaxRetries);
|
||||
throw;
|
||||
}
|
||||
|
||||
// Calculate delay and wait
|
||||
var delay = errorContext.RetryDelay ?? policy.GetDelay(attempt);
|
||||
_logger.Debug("Retry attempt {Attempt}/{MaxRetries} after {Delay}ms delay for error: {ErrorMessage}",
|
||||
attempt, policy.MaxRetries, delay.TotalMilliseconds, ex.Message);
|
||||
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
// Should not reach here, but just in case
|
||||
throw lastException ?? new InvalidOperationException("Retry loop completed without result");
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task ExecuteWithRetryAsync(
|
||||
Func<CancellationToken, Task> operation,
|
||||
RetryPolicy policy,
|
||||
Func<Exception, SynchronizerErrorContext> errorContextFactory,
|
||||
ISynchronizerErrorHandlerFactory errorHandler = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await ExecuteWithRetryAsync(
|
||||
async ct =>
|
||||
{
|
||||
await operation(ct).ConfigureAwait(false);
|
||||
return true; // Dummy return value
|
||||
},
|
||||
policy,
|
||||
errorContextFactory,
|
||||
errorHandler,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task<T> ExecuteWithRetryAsync<T>(
|
||||
Func<CancellationToken, Task<T>> operation,
|
||||
Func<Exception, SynchronizerErrorContext> errorContextFactory,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return ExecuteWithRetryAsync(
|
||||
operation,
|
||||
RetryPolicy.Default,
|
||||
errorContextFactory,
|
||||
null,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@ using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Errors;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ using System.Threading.Tasks;
|
||||
using Wino.Core.Domain.Entities.Shared;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Integration.Processors;
|
||||
using Wino.Core.Synchronizers.ImapSync;
|
||||
using Wino.Core.Synchronizers.Mail;
|
||||
|
||||
namespace Wino.Core.Services;
|
||||
@@ -17,10 +18,12 @@ public class SynchronizerFactory : ISynchronizerFactory
|
||||
private readonly IApplicationConfiguration _applicationConfiguration;
|
||||
private readonly IOutlookSynchronizerErrorHandlerFactory _outlookSynchronizerErrorHandlerFactory;
|
||||
private readonly IGmailSynchronizerErrorHandlerFactory _gmailSynchronizerErrorHandlerFactory;
|
||||
private readonly IImapSynchronizerErrorHandlerFactory _imapSynchronizerErrorHandlerFactory;
|
||||
private readonly IOutlookChangeProcessor _outlookChangeProcessor;
|
||||
private readonly IGmailChangeProcessor _gmailChangeProcessor;
|
||||
private readonly IImapChangeProcessor _imapChangeProcessor;
|
||||
private readonly IAuthenticationProvider _authenticationProvider;
|
||||
private readonly UnifiedImapSynchronizer _unifiedImapSynchronizer;
|
||||
|
||||
private readonly List<IWinoSynchronizerBase> synchronizerCache = new();
|
||||
|
||||
@@ -32,7 +35,9 @@ public class SynchronizerFactory : ISynchronizerFactory
|
||||
IImapSynchronizationStrategyProvider imapSynchronizationStrategyProvider,
|
||||
IApplicationConfiguration applicationConfiguration,
|
||||
IOutlookSynchronizerErrorHandlerFactory outlookSynchronizerErrorHandlerFactory,
|
||||
IGmailSynchronizerErrorHandlerFactory gmailSynchronizerErrorHandlerFactory)
|
||||
IGmailSynchronizerErrorHandlerFactory gmailSynchronizerErrorHandlerFactory,
|
||||
IImapSynchronizerErrorHandlerFactory imapSynchronizerErrorHandlerFactory,
|
||||
UnifiedImapSynchronizer unifiedImapSynchronizer)
|
||||
{
|
||||
_outlookChangeProcessor = outlookChangeProcessor;
|
||||
_gmailChangeProcessor = gmailChangeProcessor;
|
||||
@@ -43,6 +48,8 @@ public class SynchronizerFactory : ISynchronizerFactory
|
||||
_applicationConfiguration = applicationConfiguration;
|
||||
_outlookSynchronizerErrorHandlerFactory = outlookSynchronizerErrorHandlerFactory;
|
||||
_gmailSynchronizerErrorHandlerFactory = gmailSynchronizerErrorHandlerFactory;
|
||||
_imapSynchronizerErrorHandlerFactory = imapSynchronizerErrorHandlerFactory;
|
||||
_unifiedImapSynchronizer = unifiedImapSynchronizer;
|
||||
}
|
||||
|
||||
public async Task<IWinoSynchronizerBase> GetAccountSynchronizerAsync(Guid accountId)
|
||||
@@ -78,7 +85,7 @@ public class SynchronizerFactory : ISynchronizerFactory
|
||||
var gmailAuthenticator = _authenticationProvider.GetAuthenticator(Domain.Enums.MailProviderType.Gmail) as IGmailAuthenticator;
|
||||
return new GmailSynchronizer(mailAccount, gmailAuthenticator, _gmailChangeProcessor, _gmailSynchronizerErrorHandlerFactory);
|
||||
case Domain.Enums.MailProviderType.IMAP4:
|
||||
return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _imapSynchronizationStrategyProvider, _applicationConfiguration);
|
||||
return new ImapSynchronizer(mailAccount, _imapChangeProcessor, _imapSynchronizationStrategyProvider, _applicationConfiguration, _unifiedImapSynchronizer, _imapSynchronizerErrorHandlerFactory);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user