Refactored all synchronizers to deal with some of the chronic issues.
This commit is contained in:
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Google;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
using Wino.Core.Integration.Processors;
|
||||
|
||||
namespace Wino.Core.Synchronizers.Errors.Gmail;
|
||||
|
||||
/// <summary>
|
||||
/// Handles Gmail history ID expiration errors.
|
||||
/// When history is no longer available, resets the account's history ID to force a full resync.
|
||||
/// </summary>
|
||||
public class GmailHistoryExpiredHandler : ISynchronizerErrorHandler
|
||||
{
|
||||
private readonly ILogger _logger = Log.ForContext<GmailHistoryExpiredHandler>();
|
||||
private readonly IGmailChangeProcessor _gmailChangeProcessor;
|
||||
|
||||
public GmailHistoryExpiredHandler(IGmailChangeProcessor gmailChangeProcessor)
|
||||
{
|
||||
_gmailChangeProcessor = gmailChangeProcessor;
|
||||
}
|
||||
|
||||
public bool CanHandle(SynchronizerErrorContext error)
|
||||
{
|
||||
// Gmail returns 404 when history ID is no longer valid
|
||||
if (error.ErrorCode == 404)
|
||||
{
|
||||
var message = error.ErrorMessage?.ToLowerInvariant() ?? string.Empty;
|
||||
return message.Contains("history") || message.Contains("notfound");
|
||||
}
|
||||
|
||||
if (error.Exception is GoogleApiException googleEx)
|
||||
{
|
||||
if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
|
||||
{
|
||||
var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty;
|
||||
return errorMessage.Contains("history") ||
|
||||
errorMessage.Contains("not found") ||
|
||||
errorMessage.Contains("starthistoryid");
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public async Task<bool> HandleAsync(SynchronizerErrorContext error)
|
||||
{
|
||||
_logger.Warning(error.Exception,
|
||||
"Gmail history ID expired for account {AccountName} ({AccountId}). Resetting to force full sync.",
|
||||
error.Account?.Name, error.Account?.Id);
|
||||
|
||||
error.Severity = SynchronizerErrorSeverity.Recoverable;
|
||||
error.Category = SynchronizerErrorCategory.ResourceNotFound;
|
||||
|
||||
// Reset the account's synchronization identifier (history ID)
|
||||
if (error.Account != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(
|
||||
error.Account.Id, string.Empty).ConfigureAwait(false);
|
||||
|
||||
_logger.Information("Successfully reset Gmail history ID for account {AccountName}. Next sync will be full sync.",
|
||||
error.Account.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error(ex, "Failed to reset Gmail history ID for account {AccountName}",
|
||||
error.Account.Name);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Google;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
|
||||
namespace Wino.Core.Synchronizers.Errors.Gmail;
|
||||
|
||||
/// <summary>
|
||||
/// Handles Gmail API quota exceeded errors (HTTP 403 with quota error).
|
||||
/// This is a more severe rate limit that indicates daily quota exhaustion.
|
||||
/// </summary>
|
||||
public class GmailQuotaExceededHandler : ISynchronizerErrorHandler
|
||||
{
|
||||
private readonly ILogger _logger = Log.ForContext<GmailQuotaExceededHandler>();
|
||||
|
||||
public bool CanHandle(SynchronizerErrorContext error)
|
||||
{
|
||||
if (error.Exception is GoogleApiException googleEx)
|
||||
{
|
||||
// Quota exceeded usually returns 403
|
||||
if (googleEx.HttpStatusCode == System.Net.HttpStatusCode.Forbidden)
|
||||
{
|
||||
var errorMessage = googleEx.Message?.ToLowerInvariant() ?? string.Empty;
|
||||
var errorReason = googleEx.Error?.Errors?.FirstOrDefault()?.Reason?.ToLowerInvariant() ?? string.Empty;
|
||||
|
||||
return errorMessage.Contains("quota") ||
|
||||
errorMessage.Contains("limit exceeded") ||
|
||||
errorReason.Contains("quota") ||
|
||||
errorReason.Contains("ratelimitexceeded") ||
|
||||
errorReason.Contains("userlimitexceeded");
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public Task<bool> HandleAsync(SynchronizerErrorContext error)
|
||||
{
|
||||
_logger.Warning(error.Exception,
|
||||
"Gmail API quota exceeded for account {AccountName} ({AccountId}). Sync will be paused.",
|
||||
error.Account?.Name, error.Account?.Id);
|
||||
|
||||
// Quota exceeded is more severe - treat as fatal to prevent repeated failures
|
||||
// The user will be notified and sync will resume after quota resets
|
||||
error.Severity = SynchronizerErrorSeverity.Fatal;
|
||||
error.Category = SynchronizerErrorCategory.RateLimit;
|
||||
|
||||
// Suggest a very long delay - quotas typically reset daily
|
||||
error.RetryDelay = TimeSpan.FromHours(1);
|
||||
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using Google;
|
||||
using Serilog;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.Synchronization;
|
||||
|
||||
namespace Wino.Core.Synchronizers.Errors.Gmail;
|
||||
|
||||
/// <summary>
|
||||
/// Handles Gmail API rate limiting errors (HTTP 429 Too Many Requests).
|
||||
/// Marks the error as transient with appropriate backoff delay.
|
||||
/// </summary>
|
||||
public class GmailRateLimitHandler : ISynchronizerErrorHandler
|
||||
{
|
||||
private readonly ILogger _logger = Log.ForContext<GmailRateLimitHandler>();
|
||||
|
||||
public bool CanHandle(SynchronizerErrorContext error)
|
||||
{
|
||||
if (error.ErrorCode == 429)
|
||||
return true;
|
||||
|
||||
if (error.Exception is GoogleApiException googleEx)
|
||||
{
|
||||
return googleEx.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests ||
|
||||
(googleEx.Error?.Code == 429);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public Task<bool> HandleAsync(SynchronizerErrorContext error)
|
||||
{
|
||||
_logger.Warning(error.Exception,
|
||||
"Gmail API rate limit hit for account {AccountName} ({AccountId}). Operation: {Operation}. Will retry with backoff.",
|
||||
error.Account?.Name, error.Account?.Id, error.OperationType ?? "N/A");
|
||||
|
||||
error.Severity = SynchronizerErrorSeverity.Transient;
|
||||
error.Category = SynchronizerErrorCategory.RateLimit;
|
||||
|
||||
// Gmail rate limits are usually per-user, suggest a longer delay
|
||||
error.RetryDelay = TimeSpan.FromSeconds(10);
|
||||
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user