From 7e05d05f9492d678f0c5e2ed4ddba1c9b88d025c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kaan=20K=C3=B6se?= Date: Sat, 22 Feb 2025 23:09:53 +0100 Subject: [PATCH] Implemented cache reset for Gmail history id expiration. (#581) --- .../Enums/AccountCacheResetReason.cs | 7 ++ .../Interfaces/IAccountService.cs | 8 ++ Wino.Core.Domain/Interfaces/IMailService.cs | 6 ++ .../Interfaces/IMimeFileService.cs | 6 ++ .../Translations/en_US/resources.json | 2 + .../Services/WinoServerConnectionManager.cs | 5 +- .../Processors/DefaultChangeProcessor.cs | 9 ++ .../Processors/GmailChangeProcessor.cs | 2 + Wino.Core/Synchronizers/GmailSynchronizer.cs | 98 ++++++++++++++----- Wino.Mail.ViewModels/MailListPageViewModel.cs | 21 +++- Wino.Messages/CommunicationMessagesContext.cs | 3 + Wino.Messages/UI/AccountCacheResetMessage.cs | 7 ++ Wino.Server/ServerContext.cs | 5 +- Wino.Services/AccountService.cs | 29 ++++-- Wino.Services/MailService.cs | 14 ++- Wino.Services/MimeFileService.cs | 18 ++++ 16 files changed, 204 insertions(+), 36 deletions(-) create mode 100644 Wino.Core.Domain/Enums/AccountCacheResetReason.cs create mode 100644 Wino.Messages/UI/AccountCacheResetMessage.cs diff --git a/Wino.Core.Domain/Enums/AccountCacheResetReason.cs b/Wino.Core.Domain/Enums/AccountCacheResetReason.cs new file mode 100644 index 00000000..79c1a27e --- /dev/null +++ b/Wino.Core.Domain/Enums/AccountCacheResetReason.cs @@ -0,0 +1,7 @@ +namespace Wino.Core.Domain.Enums; + +public enum AccountCacheResetReason +{ + AccountRemoval, + ExpiredCache +} diff --git a/Wino.Core.Domain/Interfaces/IAccountService.cs b/Wino.Core.Domain/Interfaces/IAccountService.cs index 338e4270..1efc115a 100644 --- a/Wino.Core.Domain/Interfaces/IAccountService.cs +++ b/Wino.Core.Domain/Interfaces/IAccountService.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Entities.Shared; +using Wino.Core.Domain.Enums; using Wino.Core.Domain.Models.Accounts; namespace Wino.Core.Domain.Interfaces; @@ -156,4 +157,11 @@ public interface IAccountService /// Primary alias for the account. Task GetPrimaryAccountAliasAsync(Guid accountId); Task IsAccountFocusedEnabledAsync(Guid accountId); + + /// + /// Deletes mail cache in the database for the given account. + /// + /// Account id. + /// Reason for the cache reset. + Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason); } diff --git a/Wino.Core.Domain/Interfaces/IMailService.cs b/Wino.Core.Domain/Interfaces/IMailService.cs index 911640ef..e4617bba 100644 --- a/Wino.Core.Domain/Interfaces/IMailService.cs +++ b/Wino.Core.Domain/Interfaces/IMailService.cs @@ -135,4 +135,10 @@ public interface IMailService /// Mail creation package. /// Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package); + + /// + /// Checks whether the account has any draft mail locally. + /// + /// Account id. + Task HasAccountAnyDraftAsync(Guid accountId); } diff --git a/Wino.Core.Domain/Interfaces/IMimeFileService.cs b/Wino.Core.Domain/Interfaces/IMimeFileService.cs index f459e458..a7b5b8db 100644 --- a/Wino.Core.Domain/Interfaces/IMimeFileService.cs +++ b/Wino.Core.Domain/Interfaces/IMimeFileService.cs @@ -66,4 +66,10 @@ public interface IMimeFileService /// File path that physical MimeMessage is located. /// Rendering options MailRenderModel GetMailRenderModel(MimeMessage message, string mimeLocalPath, MailRenderingOptions options = null); + + /// + /// Deletes every file in the mime cache for the given account. + /// + /// Account id. + Task DeleteUserMimeCacheAsync(Guid accountId); } diff --git a/Wino.Core.Domain/Translations/en_US/resources.json b/Wino.Core.Domain/Translations/en_US/resources.json index 243151c0..f6e3fd26 100644 --- a/Wino.Core.Domain/Translations/en_US/resources.json +++ b/Wino.Core.Domain/Translations/en_US/resources.json @@ -4,6 +4,8 @@ "AccountAlias_Column_Verified": "Verified", "AccountAlias_Disclaimer_FirstLine": "Wino can only import aliases for your Gmail accounts.", "AccountAlias_Disclaimer_SecondLine": "If you want to use aliases for your Outlook or IMAP account, please add them yourself.", + "AccountCacheReset_Title": "Account Cache Reset", + "AccountCacheReset_Message": "This account requires full re-sychronization to continue working. Please wait while Wino re-synchronizes your messages...", "AccountContactNameYou": "You", "AccountCreationDialog_Completed": "all done", "AccountCreationDialog_FetchingEvents": "Fetching calendar events.", diff --git a/Wino.Core.UWP/Services/WinoServerConnectionManager.cs b/Wino.Core.UWP/Services/WinoServerConnectionManager.cs index 1d232a66..7d6d6c4d 100644 --- a/Wino.Core.UWP/Services/WinoServerConnectionManager.cs +++ b/Wino.Core.UWP/Services/WinoServerConnectionManager.cs @@ -258,7 +258,10 @@ public class WinoServerConnectionManager : WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.CopyAuthURLRequested)); break; case nameof(NewMailSynchronizationRequested): - WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson)); + WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.NewMailSynchronizationRequested)); + break; + case nameof(AccountCacheResetMessage): + WeakReferenceMessenger.Default.Send(JsonSerializer.Deserialize(messageJson, CommunicationMessagesContext.Default.AccountCacheResetMessage)); break; default: throw new Exception("Invalid data type name passed to client."); diff --git a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs index a11ae05d..cdb09724 100644 --- a/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/DefaultChangeProcessor.cs @@ -6,6 +6,7 @@ using MimeKit; using Wino.Core.Domain.Entities.Calendar; using Wino.Core.Domain.Entities.Mail; using Wino.Core.Domain.Entities.Shared; +using Wino.Core.Domain.Enums; using Wino.Core.Domain.Interfaces; using Wino.Core.Domain.Models.MailItem; using Wino.Core.Domain.Models.Synchronization; @@ -61,10 +62,12 @@ public interface IDefaultChangeProcessor Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken); Task GetMailCopyAsync(string mailCopyId); Task CreateMailRawAsync(MailAccount account, MailItemFolder mailItemFolder, NewMailItemPackage package); + Task DeleteUserMailCacheAsync(Guid accountId); } public interface IGmailChangeProcessor : IDefaultChangeProcessor { + Task HasAccountAnyDraftAsync(Guid accountId); Task MapLocalDraftAsync(string mailCopyId, string newDraftId, string newThreadId); Task CreateAssignmentAsync(Guid accountId, string mailCopyId, string remoteFolderId); Task ManageCalendarEventAsync(Event calendarEvent, AccountCalendar assignedCalendar, MailAccount organizerAccount); @@ -214,4 +217,10 @@ public class DefaultChangeProcessor(IDatabaseService databaseService, public Task UpdateCalendarDeltaSynchronizationToken(Guid calendarId, string deltaToken) => CalendarService.UpdateCalendarDeltaSynchronizationToken(calendarId, deltaToken); + + public async Task DeleteUserMailCacheAsync(Guid accountId) + { + await _mimeFileService.DeleteUserMimeCacheAsync(accountId).ConfigureAwait(false); + await AccountService.DeleteAccountMailCacheAsync(accountId, AccountCacheResetReason.ExpiredCache).ConfigureAwait(false); + } } diff --git a/Wino.Core/Integration/Processors/GmailChangeProcessor.cs b/Wino.Core/Integration/Processors/GmailChangeProcessor.cs index 428bbe1d..0c1ea2da 100644 --- a/Wino.Core/Integration/Processors/GmailChangeProcessor.cs +++ b/Wino.Core/Integration/Processors/GmailChangeProcessor.cs @@ -308,4 +308,6 @@ public class GmailChangeProcessor : DefaultChangeProcessor, IGmailChangeProcesso }; } + public Task HasAccountAnyDraftAsync(Guid accountId) + => MailService.HasAccountAnyDraftAsync(accountId); } diff --git a/Wino.Core/Synchronizers/GmailSynchronizer.cs b/Wino.Core/Synchronizers/GmailSynchronizer.cs index 29ed5063..e2d4f5c1 100644 --- a/Wino.Core/Synchronizers/GmailSynchronizer.cs +++ b/Wino.Core/Synchronizers/GmailSynchronizer.cs @@ -43,7 +43,13 @@ namespace Wino.Core.Synchronizers.Mail; public class GmailSynchronizer : WinoSynchronizer, IHttpClientFactory { public override uint BatchModificationSize => 1000; - public override uint InitialMessageDownloadCountPerFolder => 1200; + + /// + /// This is NOT the initial message download count per folder. + /// Gmail doesn't have per-folder sync. Therefore this represents to total amount that 1 page query returns until + /// there are no pages to get. Max allowed is 500. + /// + public override uint InitialMessageDownloadCountPerFolder => 500; // It's actually 100. But Gmail SDK has internal bug for Out of Memory exception. // https://github.com/googleapis/google-api-dotnet-client/issues/2603 @@ -143,6 +149,7 @@ public class GmailSynchronizer : WinoSynchronizer(); + var batchProcessCallbacks = new List>(); foreach (var batchBundle in batchedDownloadRequests) { @@ -626,8 +651,9 @@ public class GmailSynchronizer : WinoSynchronizer(request, (content, error, index, message) => { var downloadingMessageId = messageIds.ElementAt(index); + var downloadTask = HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken); - batchProcessCallbacks.Add(HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken)); + batchProcessCallbacks.Add(downloadTask); downloadedItemCount++; @@ -650,6 +676,15 @@ public class GmailSynchronizer : WinoSynchronizer a.Result).Where(a => a.HistoryId != null).Max(a => a.HistoryId.Value); + + if (maxHistoryId != 0) + { + Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, maxHistoryId.ToString()).ConfigureAwait(false); + } + } /// @@ -1100,7 +1135,7 @@ public class GmailSynchronizer : WinoSynchronizer /// /// - private async Task HandleSingleItemDownloadedCallbackAsync(Message message, + private async Task HandleSingleItemDownloadedCallbackAsync(Message message, RequestError error, string downloadingMessageId, CancellationToken cancellationToken = default) @@ -1126,7 +1161,7 @@ public class GmailSynchronizer : WinoSynchronizer currentIdentifier) + { + Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString()); + } } private async Task ProcessSingleNativeRequestResponseAsync(IRequestBundle bundle, @@ -1164,7 +1206,8 @@ public class GmailSynchronizer : WinoSynchronizer folderBundle) { @@ -1210,9 +1253,12 @@ public class GmailSynchronizer : WinoSynchronizer private async Task MapDraftIdsAsync(CancellationToken cancellationToken = default) { - // TODO: This call is not necessary if we don't have any local drafts. - // Remote drafts will be downloaded in missing message batches anyways. - // Fix it by checking whether we need to do this or not. + // Check if account has any draft locally. + // There is no point to send this query if there are no local drafts. + + bool hasLocalDrafts = await _gmailChangeProcessor.HasAccountAnyDraftAsync(Account.Id).ConfigureAwait(false); + + if (!hasLocalDrafts) return; var drafts = await _gmailService.Users.Drafts.List("me").ExecuteAsync(cancellationToken); diff --git a/Wino.Mail.ViewModels/MailListPageViewModel.cs b/Wino.Mail.ViewModels/MailListPageViewModel.cs index 08a6e602..baf7e40d 100644 --- a/Wino.Mail.ViewModels/MailListPageViewModel.cs +++ b/Wino.Mail.ViewModels/MailListPageViewModel.cs @@ -40,7 +40,8 @@ public partial class MailListPageViewModel : MailBaseViewModel, IRecipient, IRecipient, IRecipient, - IRecipient + IRecipient, + IRecipient { private bool isChangingFolder = false; @@ -1117,4 +1118,22 @@ public partial class MailListPageViewModel : MailBaseViewModel, await ExecuteUIThread(() => { IsAccountSynchronizerInSynchronization = isAnyAccountSynchronizing; }); } + + public void Receive(AccountCacheResetMessage message) + { + if (message.Reason == AccountCacheResetReason.ExpiredCache && + ActiveFolder.HandlingFolders.Any(a => a.MailAccountId == message.AccountId)) + { + var handlingFolder = ActiveFolder.HandlingFolders.FirstOrDefault(a => a.MailAccountId == message.AccountId); + + if (handlingFolder == null) return; + + _ = ExecuteUIThread(() => + { + MailCollection.Clear(); + + _mailDialogService.InfoBarMessage(Translator.AccountCacheReset_Title, Translator.AccountCacheReset_Message, InfoBarMessageType.Warning); + }); + } + } } diff --git a/Wino.Messages/CommunicationMessagesContext.cs b/Wino.Messages/CommunicationMessagesContext.cs index 49719f77..ed059f2f 100644 --- a/Wino.Messages/CommunicationMessagesContext.cs +++ b/Wino.Messages/CommunicationMessagesContext.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using Wino.Messaging.Server; using Wino.Messaging.UI; namespace Wino.Messaging; @@ -23,4 +24,6 @@ namespace Wino.Messaging; [JsonSerializable(typeof(AccountSynchronizationProgressUpdatedMessage))] [JsonSerializable(typeof(AccountFolderConfigurationUpdated))] [JsonSerializable(typeof(CopyAuthURLRequested))] +[JsonSerializable(typeof(NewMailSynchronizationRequested))] +[JsonSerializable(typeof(AccountCacheResetMessage))] public partial class CommunicationMessagesContext : JsonSerializerContext; diff --git a/Wino.Messages/UI/AccountCacheResetMessage.cs b/Wino.Messages/UI/AccountCacheResetMessage.cs new file mode 100644 index 00000000..04a618b7 --- /dev/null +++ b/Wino.Messages/UI/AccountCacheResetMessage.cs @@ -0,0 +1,7 @@ +using System; +using Wino.Core.Domain.Enums; + +namespace Wino.Messaging.UI; + +// Raised when the account's mail cache is reset. +public record AccountCacheResetMessage(Guid AccountId, AccountCacheResetReason Reason) : UIMessageBase; diff --git a/Wino.Server/ServerContext.cs b/Wino.Server/ServerContext.cs index 46dd634d..79b2454e 100644 --- a/Wino.Server/ServerContext.cs +++ b/Wino.Server/ServerContext.cs @@ -44,7 +44,8 @@ public class ServerContext : IRecipient, IRecipient, IRecipient, - IRecipient + IRecipient, + IRecipient { private readonly System.Timers.Timer _timer; private static object connectionLock = new object(); @@ -147,6 +148,8 @@ public class ServerContext : public async void Receive(OnlineSearchRequested message) => await SendMessageAsync(MessageType.UIMessage, message); + public async void Receive(AccountCacheResetMessage message) => await SendMessageAsync(MessageType.UIMessage, message); + #endregion private string GetAppPackagFamilyName() diff --git a/Wino.Services/AccountService.cs b/Wino.Services/AccountService.cs index fa8393c4..f0629884 100644 --- a/Wino.Services/AccountService.cs +++ b/Wino.Services/AccountService.cs @@ -22,15 +22,18 @@ public class AccountService : BaseDatabaseService, IAccountService public IAuthenticator ExternalAuthenticationAuthenticator { get; set; } private readonly ISignatureService _signatureService; + private readonly IMimeFileService _mimeFileService; private readonly IPreferencesService _preferencesService; private readonly ILogger _logger = Log.ForContext(); public AccountService(IDatabaseService databaseService, ISignatureService signatureService, + IMimeFileService mimeFileService, IPreferencesService preferencesService) : base(databaseService) { _signatureService = signatureService; + _mimeFileService = mimeFileService; _preferencesService = preferencesService; } @@ -262,12 +265,26 @@ public class AccountService : BaseDatabaseService, IAccountService private Task GetMergedInboxInformationAsync(Guid mergedInboxId) => Connection.Table().FirstOrDefaultAsync(a => a.Id == mergedInboxId); + public async Task DeleteAccountMailCacheAsync(Guid accountId, AccountCacheResetReason accountCacheResetReason) + { + var deleteQuery = new Query("MailCopy") + .WhereIn("Id", q => q + .From("MailCopy") + .Select("Id") + .WhereIn("FolderId", q2 => q2 + .From("MailItemFolder") + .Select("Id") + .Where("MailAccountId", accountId) + )).AsDelete(); + + await Connection.ExecuteAsync(deleteQuery.GetRawQuery()); + + WeakReferenceMessenger.Default.Send(new AccountCacheResetMessage(accountId, accountCacheResetReason)); + } + public async Task DeleteAccountAsync(MailAccount account) { - // TODO: Delete mime messages and attachments. - // TODO: Delete token cache by underlying provider. - - await Connection.ExecuteAsync("DELETE FROM MailCopy WHERE Id IN(SELECT Id FROM MailCopy WHERE FolderId IN (SELECT Id from MailItemFolder WHERE MailAccountId == ?))", account.Id); + await DeleteAccountMailCacheAsync(account.Id, AccountCacheResetReason.AccountRemoval); await Connection.Table().DeleteAsync(a => a.MailAccountId == account.Id); await Connection.Table().DeleteAsync(a => a.MailAccountId == account.Id); @@ -302,6 +319,8 @@ public class AccountService : BaseDatabaseService, IAccountService await Connection.DeleteAsync(account); + await _mimeFileService.DeleteUserMimeCacheAsync(account.Id).ConfigureAwait(false); + // Clear out or set up a new startup entity id. // Next account after the deleted one will be the startup account. @@ -319,8 +338,6 @@ public class AccountService : BaseDatabaseService, IAccountService } } - - ReportUIChange(new AccountRemovedMessage(account)); } diff --git a/Wino.Services/MailService.cs b/Wino.Services/MailService.cs index 983fc8a4..054a781d 100644 --- a/Wino.Services/MailService.cs +++ b/Wino.Services/MailService.cs @@ -119,6 +119,18 @@ public class MailService : BaseDatabaseService, IMailService return mails; } + public async Task HasAccountAnyDraftAsync(Guid accountId) + { + // Get the draft folder. + var draftFolder = await _folderService.GetSpecialFolderByAccountIdAsync(accountId, SpecialFolderType.Draft); + + if (draftFolder == null) return false; + + var draftCount = await Connection.Table().Where(a => a.FolderId == draftFolder.Id).CountAsync(); + + return draftCount > 0; + } + public async Task> GetUnreadMailsByFolderIdAsync(Guid folderId) { var unreadMails = await Connection.QueryAsync("SELECT * FROM MailCopy WHERE FolderId = ? AND IsRead = 0", folderId); @@ -143,7 +155,7 @@ public class MailService : BaseDatabaseService, IMailService //} // SQLite PCL doesn't support joins. - // We make the query using SqlKatka and execute it directly on SQLite-PCL. + // We make the query using SqlKata and execute it directly on SQLite-PCL. var query = new Query("MailCopy") .Join("MailItemFolder", "MailCopy.FolderId", "MailItemFolder.Id") diff --git a/Wino.Services/MimeFileService.cs b/Wino.Services/MimeFileService.cs index c22bc9ec..29839350 100644 --- a/Wino.Services/MimeFileService.cs +++ b/Wino.Services/MimeFileService.cs @@ -176,4 +176,22 @@ public class MimeFileService : IMimeFileService return renderingModel; } + + public async Task DeleteUserMimeCacheAsync(Guid accountId) + { + var mimeFolderPath = await _nativeAppService.GetMimeMessageStoragePath().ConfigureAwait(false); + var mimeDirectory = Path.Combine(mimeFolderPath, accountId.ToString()); + + try + { + if (Directory.Exists(mimeDirectory)) + { + Directory.Delete(mimeDirectory, true); + } + } + catch (Exception ex) + { + Log.Error(ex, "Failed to remove user's mime cache folder."); + } + } }