javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented cache reset for Gmail history id expiration. (#581)

Browse Source
This commit is contained in:
Burak Kaan Köse
2025-02-22 23:09:53 +01:00
committed by GitHub
parent bd5b51c62f
commit 7e05d05f94
16 changed files with 204 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Wino.Core.Domain/Enums/AccountCacheResetReason.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace Wino.Core.Domain.Enums;
public enum AccountCacheResetReason
{
AccountRemoval,
ExpiredCache
}

8
Wino.Core.Domain/Interfaces/IAccountService.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Entities.Shared;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Models.Accounts;
namespace Wino.Core.Domain.Interfaces;
@@ -156,4 +157,11 @@ public interface IAccountService
/// <returns>Primary alias for the account.</returns>
Task<MailAccountAlias> GetPrimaryAccountAliasAsync(Guid accountId);
Task<bool> IsAccountFocusedEnabledAsync(Guid accountId);
/// <summary>
/// Deletes mail cache in the database for the given account.
/// </summary>
/// <param name="accountId">Account id.</param>
/// <param name="AccountCacheResetReason">Reason for the cache reset.</param>
Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason);
}

6
Wino.Core.Domain/Interfaces/IMailService.cs
View File

@@ -135,4 +135,10 @@ public interface IMailService
/// <param name="package">Mail creation package.</param>
/// <returns></returns>
Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
/// <summary>
/// Checks whether the account has any draft mail locally.
/// </summary>
/// <param name="accountId">Account id.</param>
Task<bool> HasAccountAnyDraftAsync(Guid accountId);
}

6
Wino.Core.Domain/Interfaces/IMimeFileService.cs
View File

@@ -66,4 +66,10 @@ public interface IMimeFileService
/// <param name="mimeLocalPath">File path that physical MimeMessage is located.</param>
/// <param name="options">Rendering options</param>
MailRenderModel GetMailRenderModel(MimeMessage message, string mimeLocalPath, MailRenderingOptions options = null);
/// <summary>
/// Deletes every file in the mime cache for the given account.
/// </summary>
/// <param name="accountId">Account id.</param>
Task DeleteUserMimeCacheAsync(Guid accountId);
}

2
Wino.Core.Domain/Translations/en_US/resources.json
View File

@@ -4,6 +4,8 @@
"AccountAlias_Column_Verified": "Verified",
"AccountAlias_Disclaimer_FirstLine": "Wino can only import aliases for your Gmail accounts.",
"AccountAlias_Disclaimer_SecondLine": "If you want to use aliases for your Outlook or IMAP account, please add them yourself.",
"AccountCacheReset_Title": "Account Cache Reset",
"AccountCacheReset_Message": "This account requires full re-sychronization to continue working. Please wait while Wino re-synchronizes your messages...",
"AccountContactNameYou": "You",
"AccountCreationDialog_Completed": "all done",
"AccountCreationDialog_FetchingEvents": "Fetching calendar events.",

5
Wino.Core.UWP/Services/WinoServerConnectionManager.cs
View File

@@ -258,7 +258,10 @@ public class WinoServerConnectionManager :
WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.CopyAuthURLRequested));
break;
case nameof(NewMailSynchronizationRequested):
WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize<NewMailSynchronizationRequested>(messageJson));
WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.NewMailSynchronizationRequested));
break;
case nameof(AccountCacheResetMessage):
WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.AccountCacheResetMessage));
break;
default:
throw new Exception("Invalid data type name passed to client.");

9
Wino.Core/Integration/Processors/DefaultChangeProcessor.cs
View File

@@ -6,6 +6,7 @@ using MimeKit;
using Wino.Core.Domain.Entities.Calendar;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Entities.Shared;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
@@ -61,10 +62,12 @@ public interface IDefaultChangeProcessor
Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken);
Task<MailCopy> GetMailCopyAsync(string mailCopyId);
Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package);
Task DeleteUserMailCacheAsync(Guid accountId);
}
public interface IGmailChangeProcessor : IDefaultChangeProcessor
{
Task<bool> HasAccountAnyDraftAsync(Guid accountId);
Task MapLocalDraftAsync(string mailCopyId, string newDraftId, string newThreadId);
Task CreateAssignmentAsync(Guid accountId, string mailCopyId, string remoteFolderId);
Task ManageCalendarEventAsync(Event calendarEvent, AccountCalendar assignedCalendar, MailAccount organizerAccount);
@@ -214,4 +217,10 @@ public class DefaultChangeProcessor(IDatabaseService databaseService,
public Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken)
=> CalendarService.UpdateCalendarDeltaSynchronizationToken(calendarId, deltaToken);
public async Task DeleteUserMailCacheAsync(Guid accountId)
{
await _mimeFileService.DeleteUserMimeCacheAsync(accountId).ConfigureAwait(false);
await AccountService.DeleteAccountMailCacheAsync(accountId, AccountCacheResetReason.ExpiredCache).ConfigureAwait(false);
}
}

2
Wino.Core/Integration/Processors/GmailChangeProcessor.cs
View File

@@ -308,4 +308,6 @@ public class GmailChangeProcessor : DefaultChangeProcessor, IGmailChangeProcesso
};
}
public Task<bool> HasAccountAnyDraftAsync(Guid accountId)
=> MailService.HasAccountAnyDraftAsync(accountId);
}

98
Wino.Core/Synchronizers/GmailSynchronizer.cs
View File

@@ -43,7 +43,13 @@ namespace Wino.Core.Synchronizers.Mail;
public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message, Event>, IHttpClientFactory
{
public override uint BatchModificationSize => 1000;
public override uint InitialMessageDownloadCountPerFolder => 1200;
/// <summary>
/// This is NOT the initial message download count per folder.
/// Gmail doesn't have per-folder sync. Therefore this represents to total amount that 1 page query returns until
/// there are no pages to get. Max allowed is 500.
/// </summary>
public override uint InitialMessageDownloadCountPerFolder => 500;
// It's actually 100. But Gmail SDK has internal bug for Out of Memory exception.
// https://github.com/googleapis/google-api-dotnet-client/issues/2603
@@ -143,6 +149,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
if (options.Type == MailSynchronizationType.FoldersOnly) return MailSynchronizationResult.Empty;
retry:
cancellationToken.ThrowIfCancellationRequested();
bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier);
@@ -204,25 +211,43 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
var historyRequest = _gmailService.Users.History.List("me");
historyRequest.StartHistoryId = startHistoryId;
while (!string.IsNullOrEmpty(nextPageToken))
try
{
// If this is the first delta check, start from the last history id.
// Otherwise start from the next page token. We set them both to the same value for start.
// For each different page we set the page token to the next page token.
while (!string.IsNullOrEmpty(nextPageToken))
{
// If this is the first delta check, start from the last history id.
// Otherwise start from the next page token. We set them both to the same value for start.
// For each different page we set the page token to the next page token.
bool isFirstDeltaCheck = nextPageToken == startHistoryId.ToString();
bool isFirstDeltaCheck = nextPageToken == startHistoryId.ToString();
if (!isFirstDeltaCheck)
historyRequest.PageToken = nextPageToken;
if (!isFirstDeltaCheck)
historyRequest.PageToken = nextPageToken;
var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
nextPageToken = historyResponse.NextPageToken;
nextPageToken = historyResponse.NextPageToken;
if (historyResponse.History == null)
continue;
if (historyResponse.History == null)
continue;
deltaChanges.Add(historyResponse);
deltaChanges.Add(historyResponse);
}
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
{
// History ID is too old or expired, need to do a full sync.
// Theoratically we need to delete the local cache and start from scratch.
_logger.Warning("History ID {StartHistoryId} is expired for {Name}. Will remove user's mail cache and do full sync.", startHistoryId, Account.Name);
await _gmailChangeProcessor.DeleteUserMailCacheAsync(Account.Id).ConfigureAwait(false);
Account.SynchronizationDeltaIdentifier = string.Empty;
await _gmailChangeProcessor.UpdateAccountAsync(Account).ConfigureAwait(false);
goto retry;
}
}
@@ -612,7 +637,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
// async callback is finished. Therefore we need to wrap all local database processings into task list and wait all of them to finish
// Batch execution finishes after response parsing is done.
var batchProcessCallbacks = new List<Task>();
var batchProcessCallbacks = new List<Task<Message>>();
foreach (var batchBundle in batchedDownloadRequests)
{
@@ -626,8 +651,9 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
batchRequest.Queue<Message>(request, (content, error, index, message) =>
{
var downloadingMessageId = messageIds.ElementAt(index);
var downloadTask = HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken);
batchProcessCallbacks.Add(HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken));
batchProcessCallbacks.Add(downloadTask);
downloadedItemCount++;
@@ -650,6 +676,15 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
// Wait for all processing to finish.
await Task.WhenAll(batchProcessCallbacks).ConfigureAwait(false);
// Try to update max history id.
var maxHistoryId = batchProcessCallbacks.Select(a => a.Result).Where(a => a.HistoryId != null).Max(a => a.HistoryId.Value);
if (maxHistoryId != 0)
{
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, maxHistoryId.ToString()).ConfigureAwait(false);
}
}
/// <summary>
@@ -1100,7 +1135,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
/// <param name="error"></param>
/// <param name="httpResponseMessage"></param>
/// <param name="cancellationToken"></param>
private async Task HandleSingleItemDownloadedCallbackAsync(Message message,
private async Task<Message> HandleSingleItemDownloadedCallbackAsync(Message message,
RequestError error,
string downloadingMessageId,
CancellationToken cancellationToken = default)
@@ -1126,7 +1161,7 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
{
_logger.Warning("Skipped GMail message download for {DownloadingMessageId}", downloadingMessageId);
return;
return null;
}
// Gmail has LabelId property for each message.
@@ -1135,20 +1170,27 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
// If CreateNewMailPackagesAsync returns null it means local draft mapping is done.
// We don't need to insert anything else.
if (mailPackage == null)
return;
if (mailPackage == null) return message;
foreach (var package in mailPackage)
{
await _gmailChangeProcessor.CreateMailAsync(Account.Id, package).ConfigureAwait(false);
}
return message;
}
private async Task UpdateAccountSyncIdentifierFromMessageAsync(Message message)
{
// Try updating the history change identifier if any.
if (message.HistoryId == null) return;
// Delta changes also has history id but the maximum id is preserved in the account service.
// TODO: This is not good. Centralize the identifier fetch and prevent direct access here.
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString());
if (ulong.TryParse(Account.SynchronizationDeltaIdentifier, out ulong currentIdentifier) &&
ulong.TryParse(message.HistoryId.Value.ToString(), out ulong messageIdentifier) &&
messageIdentifier > currentIdentifier)
{
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString());
}
}
private async Task ProcessSingleNativeRequestResponseAsync(IRequestBundle<IClientServiceRequest> bundle,
@@ -1164,7 +1206,8 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
if (gmailMessage == null) return;
await HandleSingleItemDownloadedCallbackAsync(gmailMessage, error, "unknown", cancellationToken);
await HandleSingleItemDownloadedCallbackAsync(gmailMessage, error, string.Empty, cancellationToken);
await UpdateAccountSyncIdentifierFromMessageAsync(gmailMessage).ConfigureAwait(false);
}
else if (bundle is HttpRequestBundle<IClientServiceRequest, Label> folderBundle)
{
@@ -1210,9 +1253,12 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
/// </summary>
private async Task MapDraftIdsAsync(CancellationToken cancellationToken = default)
{
// TODO: This call is not necessary if we don't have any local drafts.
// Remote drafts will be downloaded in missing message batches anyways.
// Fix it by checking whether we need to do this or not.
// Check if account has any draft locally.
// There is no point to send this query if there are no local drafts.
bool hasLocalDrafts = await _gmailChangeProcessor.HasAccountAnyDraftAsync(Account.Id).ConfigureAwait(false);
if (!hasLocalDrafts) return;
var drafts = await _gmailService.Users.Drafts.List("me").ExecuteAsync(cancellationToken);

21
Wino.Mail.ViewModels/MailListPageViewModel.cs
View File

@@ -40,7 +40,8 @@ public partial class MailListPageViewModel : MailBaseViewModel,
IRecipient<MailItemSelectionRemovedEvent>,
IRecipient<AccountSynchronizationCompleted>,
IRecipient<NewMailSynchronizationRequested>,
IRecipient<AccountSynchronizerStateChanged>
IRecipient<AccountSynchronizerStateChanged>,
IRecipient<AccountCacheResetMessage>
{
private bool isChangingFolder = false;
@@ -1117,4 +1118,22 @@ public partial class MailListPageViewModel : MailBaseViewModel,
await ExecuteUIThread(() => { IsAccountSynchronizerInSynchronization = isAnyAccountSynchronizing; });
}
public void Receive(AccountCacheResetMessage message)
{
if (message.Reason == AccountCacheResetReason.ExpiredCache &&
ActiveFolder.HandlingFolders.Any(a => a.MailAccountId == message.AccountId))
{
var handlingFolder = ActiveFolder.HandlingFolders.FirstOrDefault(a => a.MailAccountId == message.AccountId);
if (handlingFolder == null) return;
_ = ExecuteUIThread(() =>
{
MailCollection.Clear();
_mailDialogService.InfoBarMessage(Translator.AccountCacheReset_Title, Translator.AccountCacheReset_Message, InfoBarMessageType.Warning);
});
}
}
}

3
Wino.Messages/CommunicationMessagesContext.cs
View File

@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
using Wino.Messaging.Server;
using Wino.Messaging.UI;
namespace Wino.Messaging;
@@ -23,4 +24,6 @@ namespace Wino.Messaging;
[JsonSerializable(typeof(AccountSynchronizationProgressUpdatedMessage))]
[JsonSerializable(typeof(AccountFolderConfigurationUpdated))]
[JsonSerializable(typeof(CopyAuthURLRequested))]
[JsonSerializable(typeof(NewMailSynchronizationRequested))]
[JsonSerializable(typeof(AccountCacheResetMessage))]
public partial class CommunicationMessagesContext : JsonSerializerContext;

7
Wino.Messages/UI/AccountCacheResetMessage.cs Normal file
View File

@@ -0,0 +1,7 @@
using System;
using Wino.Core.Domain.Enums;
namespace Wino.Messaging.UI;
// Raised when the account's mail cache is reset.
public record AccountCacheResetMessage(Guid AccountId, AccountCacheResetReason Reason) : UIMessageBase<AccountCacheResetMessage>;

5
Wino.Server/ServerContext.cs
View File

@@ -44,7 +44,8 @@ public class ServerContext :
IRecipient<AccountFolderConfigurationUpdated>,
IRecipient<CopyAuthURLRequested>,
IRecipient<NewMailSynchronizationRequested>,
IRecipient<OnlineSearchRequested>
IRecipient<OnlineSearchRequested>,
IRecipient<AccountCacheResetMessage>
{
private readonly System.Timers.Timer _timer;
private static object connectionLock = new object();
@@ -147,6 +148,8 @@ public class ServerContext :
public async void Receive(OnlineSearchRequested message) => await SendMessageAsync(MessageType.UIMessage, message);
public async void Receive(AccountCacheResetMessage message) => await SendMessageAsync(MessageType.UIMessage, message);
#endregion
private string GetAppPackagFamilyName()

29
Wino.Services/AccountService.cs
View File

@@ -22,15 +22,18 @@ public class AccountService : BaseDatabaseService, IAccountService
public IAuthenticator ExternalAuthenticationAuthenticator { get; set; }
private readonly ISignatureService _signatureService;
private readonly IMimeFileService _mimeFileService;
private readonly IPreferencesService _preferencesService;
private readonly ILogger _logger = Log.ForContext<AccountService>();
public AccountService(IDatabaseService databaseService,
ISignatureService signatureService,
IMimeFileService mimeFileService,
IPreferencesService preferencesService) : base(databaseService)
{
_signatureService = signatureService;
_mimeFileService = mimeFileService;
_preferencesService = preferencesService;
}
@@ -262,12 +265,26 @@ public class AccountService : BaseDatabaseService, IAccountService
private Task<MergedInbox> GetMergedInboxInformationAsync(Guid mergedInboxId)
=> Connection.Table<MergedInbox>().FirstOrDefaultAsync(a => a.Id == mergedInboxId);
public async Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason)
{
var deleteQuery = new Query("MailCopy")
.WhereIn("Id", q => q
.From("MailCopy")
.Select("Id")
.WhereIn("FolderId", q2 => q2
.From("MailItemFolder")
.Select("Id")
.Where("MailAccountId", accountId)
)).AsDelete();
await Connection.ExecuteAsync(deleteQuery.GetRawQuery());
WeakReferenceMessenger.Default.Send(new AccountCacheResetMessage(accountId, accountCacheResetReason));
}
public async Task DeleteAccountAsync(MailAccount account)
{
// TODO: Delete mime messages and attachments.
// TODO: Delete token cache by underlying provider.
await Connection.ExecuteAsync("DELETE FROM MailCopy WHERE Id IN(SELECT Id FROM MailCopy WHERE FolderId IN (SELECT Id from MailItemFolder WHERE MailAccountId == ?))", account.Id);
await DeleteAccountMailCacheAsync(account.Id, AccountCacheResetReason.AccountRemoval);
await Connection.Table<MailItemFolder>().DeleteAsync(a => a.MailAccountId == account.Id);
await Connection.Table<AccountSignature>().DeleteAsync(a => a.MailAccountId == account.Id);
@@ -302,6 +319,8 @@ public class AccountService : BaseDatabaseService, IAccountService
await Connection.DeleteAsync(account);
await _mimeFileService.DeleteUserMimeCacheAsync(account.Id).ConfigureAwait(false);
// Clear out or set up a new startup entity id.
// Next account after the deleted one will be the startup account.
@@ -319,8 +338,6 @@ public class AccountService : BaseDatabaseService, IAccountService
}
}
ReportUIChange(new AccountRemovedMessage(account));
}

14
Wino.Services/MailService.cs
View File

@@ -119,6 +119,18 @@ public class MailService : BaseDatabaseService, IMailService
return mails;
}
public async Task<bool> HasAccountAnyDraftAsync(Guid accountId)
{
// Get the draft folder.
var draftFolder = await _folderService.GetSpecialFolderByAccountIdAsync(accountId, SpecialFolderType.Draft);
if (draftFolder == null) return false;
var draftCount = await Connection.Table<MailCopy>().Where(a => a.FolderId == draftFolder.Id).CountAsync();
return draftCount > 0;
}
public async Task<List<MailCopy>> GetUnreadMailsByFolderIdAsync(Guid folderId)
{
var unreadMails = await Connection.QueryAsync<MailCopy>("SELECT * FROM MailCopy WHERE FolderId = ? AND IsRead = 0", folderId);
@@ -143,7 +155,7 @@ public class MailService : BaseDatabaseService, IMailService
//}
// SQLite PCL doesn't support joins.
// We make the query using SqlKatka and execute it directly on SQLite-PCL.
// We make the query using SqlKata and execute it directly on SQLite-PCL.
var query = new Query("MailCopy")
.Join("MailItemFolder", "MailCopy.FolderId", "MailItemFolder.Id")

18
Wino.Services/MimeFileService.cs
View File

@@ -176,4 +176,22 @@ public class MimeFileService : IMimeFileService
return renderingModel;
}
public async Task DeleteUserMimeCacheAsync(Guid accountId)
{
var mimeFolderPath = await _nativeAppService.GetMimeMessageStoragePath().ConfigureAwait(false);
var mimeDirectory = Path.Combine(mimeFolderPath, accountId.ToString());
try
{
if (Directory.Exists(mimeDirectory))
{
Directory.Delete(mimeDirectory, true);
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to remove user's mime cache folder.");
}
}
}