From 2040d4abceab17b458f952e906c008b2ad8a98dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Kaan=20K=C3=B6se?= Date: Sun, 1 Mar 2026 09:14:02 +0100 Subject: [PATCH] Optimize mail fetching with batch DB queries and in-memory caching (#827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: batch-load folders, accounts, and contacts in FetchMailsAsync Replace the sequential per-mail property-loading loop with a three-step batch pre-load strategy, eliminating the N+1 DB call pattern that was the main bottleneck when building the mail list with threading enabled. Changes: - Pre-seed the folder cache from MailListInitializationOptions.Folders so that the most common folders (inbox, sent, etc.) never trigger a DB lookup at all. - Load all accounts in a single GetAccountsAsync() call instead of one GetAccountAsync() call per mail (typically 1–5 accounts total). - Fetch all sender contacts in a single SQL IN(...) query via the new GetContactsByAddressesAsync() method instead of one query per address. - Property assignment is now fully synchronous (no awaits in the loop) since all data is pre-loaded into plain Dictionary. - Thread-expansion follows the same pattern: new folder IDs are loaded in parallel via Task.WhenAll; new contact addresses are batch-fetched with a second IN(...) query. - Also apply batch pre-loading to GetMailItemsAsync (used by merge-inbox sync path) which had the same sequential issue. - Remove the now-unused LoadAssignedPropertiesWithCacheAsync helper and the ConcurrentDictionary dependency it required. - Tighten GetMailsByThreadIdsAsync to skip the Id NOT IN clause entirely when the exclusion set is empty. https://claude.ai/code/session_018bqahGc6zi95JJhc2aARKS * test: add MailFetchingTests with correctness and performance coverage Adds integration tests for MailService.FetchMailsAsync that exercise the full real-service stack (MailService → FolderService / AccountService / ContactService) backed by the shared in-memory SQLite helper. Four tests are included: • ExpandsSiblingsOutsidePage – proves thread expansion fetches mails that fall beyond the initial SQL page (6 mails, page=4, expects 6 returned). • NeverExpandsSiblings – proves threading is truly opt-in; with CreateThreads=false the result exactly matches the raw page size. • ResolvesFromAllThreeSources – verifies contact resolution for a known contact (from the AccountContact table), an unknown sender (ad-hoc fallback), and a self-sent mail (built from account metadata). • 1000Mails_70Threads_CompletesWithinBudget – the performance scenario: 1 000 mails (70 threads × 7 + 510 standalone), 40 rotating sender addresses (20 with DB contacts). Times and reports two scenarios: - Default first-page fetch (100 mails) + expansion of one partial thread (expects > 100 mails returned). - Full load of all 1 000 mails with threading enabled (expects exactly 1 000 mails returned, all 70 threads intact, < 5 s). Elapsed times for both scenarios are written to xUnit test output so they appear in CI logs and can be tracked across builds. https://claude.ai/code/session_018bqahGc6zi95JJhc2aARKS --------- Co-authored-by: Claude --- .../Interfaces/IContactService.cs | 1 + Wino.Core.Tests/Services/MailFetchingTests.cs | 381 ++++++++++++++++++ Wino.Services/ContactService.cs | 12 + Wino.Services/MailService.cs | 253 +++++++----- 4 files changed, 539 insertions(+), 108 deletions(-) create mode 100644 Wino.Core.Tests/Services/MailFetchingTests.cs diff --git a/Wino.Core.Domain/Interfaces/IContactService.cs b/Wino.Core.Domain/Interfaces/IContactService.cs index f899d7d0..b70bb957 100644 --- a/Wino.Core.Domain/Interfaces/IContactService.cs +++ b/Wino.Core.Domain/Interfaces/IContactService.cs @@ -11,6 +11,7 @@ public interface IContactService { Task> GetAddressInformationAsync(string queryText); Task GetAddressInformationByAddressAsync(string address); + Task> GetContactsByAddressesAsync(IEnumerable addresses); Task SaveAddressInformationAsync(MimeMessage message); Task SaveAddressInformationAsync(IEnumerable contacts); Task CreateNewContactAsync(string address, string displayName); diff --git a/Wino.Core.Tests/Services/MailFetchingTests.cs b/Wino.Core.Tests/Services/MailFetchingTests.cs new file mode 100644 index 00000000..ebf973e5 --- /dev/null +++ b/Wino.Core.Tests/Services/MailFetchingTests.cs @@ -0,0 +1,381 @@ +using System.Diagnostics; +using FluentAssertions; +using Moq; +using Wino.Core.Domain.Entities.Mail; +using Wino.Core.Domain.Entities.Shared; +using Wino.Core.Domain.Enums; +using Wino.Core.Domain.Interfaces; +using Wino.Core.Domain.Models.MailItem; +using Wino.Core.Tests.Helpers; +using Wino.Services; +using Xunit; +using Xunit.Abstractions; + +namespace Wino.Core.Tests.Services; + +/// +/// Integration tests for MailService.FetchMailsAsync that verify the correctness of +/// thread expansion and contact resolution, and track performance for large inboxes. +/// +/// All tests run against a real in-memory SQLite file via the full service stack +/// (MailService → FolderService / AccountService / ContactService) so that the +/// batch-query path introduced in the performance optimisation is exercised end-to-end. +/// +public class MailFetchingTests : IAsyncLifetime +{ + // ── Infrastructure ───────────────────────────────────────────────────────── + + private readonly ITestOutputHelper _output; + private InMemoryDatabaseService _databaseService = null!; + private MailService _mailService = null!; + private MailAccount _testAccount = null!; + private MailItemFolder _inboxFolder = null!; + + public MailFetchingTests(ITestOutputHelper output) + { + _output = output; + } + + public async Task InitializeAsync() + { + _databaseService = new InMemoryDatabaseService(); + await _databaseService.InitializeAsync(); + + _testAccount = new MailAccount + { + Id = Guid.NewGuid(), + Name = "Test Account", + Address = "me@test.local", + SenderName = "Test User", + ProviderType = MailProviderType.IMAP4 + }; + + _inboxFolder = new MailItemFolder + { + Id = Guid.NewGuid(), + MailAccountId = _testAccount.Id, + FolderName = "Inbox", + SpecialFolderType = SpecialFolderType.Inbox, + IsSystemFolder = true, + IsSynchronizationEnabled = true + }; + + await _databaseService.Connection.InsertAsync(_testAccount, typeof(MailAccount)); + await _databaseService.Connection.InsertAsync(_inboxFolder, typeof(MailItemFolder)); + + _mailService = BuildMailService(_databaseService); + } + + public async Task DisposeAsync() => await _databaseService.DisposeAsync(); + + // ── Correctness: threading ON ────────────────────────────────────────────── + + /// + /// Verifies that thread siblings which fall outside the initial SQL page are + /// fetched by the expansion step, so every thread is always fully represented. + /// + /// Setup: 2 threads of 3 mails each (6 mails total), page size = 4. + /// The main query retrieves Thread A (3 mails, newest) and Thread B mail 1 (position 4). + /// Thread expansion must then fetch Thread B mails 2-3 that were beyond the page. + /// + [Fact] + public async Task FetchMailsAsync_WithThreadingEnabled_ExpandsSiblingsOutsidePage() + { + const int PageSize = 4; + var threadA = Guid.NewGuid().ToString(); + var threadB = Guid.NewGuid().ToString(); + var baseDate = DateTime.UtcNow; + + var mails = new List + { + // Thread A – all 3 land within the first page (positions 1–3) + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-1), threadId: threadA), + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-2), threadId: threadA), + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-3), threadId: threadA), + // Thread B – only position 4 lands in the page; 5 and 6 must be expanded + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-4), threadId: threadB), + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-5), threadId: threadB), + BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-6), threadId: threadB) + }; + await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy)); + + var options = BuildOptions([_inboxFolder], createThreads: true, take: PageSize); + + // Act + var result = await _mailService.FetchMailsAsync(options); + + // Assert – all 6 mails returned even though the page only held 4 + result.Should().HaveCount(6, + "the 2 Thread B siblings outside the initial page must be fetched by expansion"); + result.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null, + "every returned mail must have its account and folder resolved"); + result.Count(m => m.ThreadId == threadA).Should().Be(3, "Thread A must be complete"); + result.Count(m => m.ThreadId == threadB).Should().Be(3, "Thread B must be complete"); + } + + // ── Correctness: threading OFF ───────────────────────────────────────────── + + /// + /// Verifies that when threading is disabled the result exactly matches the raw + /// SQL page — no sibling expansion occurs. + /// + [Fact] + public async Task FetchMailsAsync_WithThreadingDisabled_NeverExpandsSiblings() + { + const int PageSize = 4; + var threadId = Guid.NewGuid().ToString(); + var baseDate = DateTime.UtcNow; + + // 6 mails all sharing a ThreadId; with threading OFF only the first 4 come back + var mails = Enumerable.Range(0, 6) + .Select(i => BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-i), threadId: threadId)) + .ToList(); + + await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy)); + + var options = BuildOptions([_inboxFolder], createThreads: false, take: PageSize); + + // Act + var result = await _mailService.FetchMailsAsync(options); + + // Assert – exactly the page size; no expansion happened + result.Should().HaveCount(PageSize, + "with threading disabled the result must match the raw page size"); + result.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null); + } + + // ── Correctness: contact resolution ─────────────────────────────────────── + + /// + /// Verifies that sender contacts are resolved from three distinct paths: + /// the contact store (known sender), the unknown-sender fallback, and the + /// account-metadata shortcut used for self-sent mails. + /// + [Fact] + public async Task FetchMailsAsync_SenderContact_ResolvesFromAllThreeSources() + { + const string KnownAddress = "known@example.com"; + const string UnknownAddress = "unknown@example.com"; + + await _databaseService.Connection.InsertAsync( + new AccountContact { Address = KnownAddress, Name = "Known Sender" }, + typeof(AccountContact)); + + var mails = new List + { + BuildMail(_inboxFolder.Id, DateTime.UtcNow, fromAddress: KnownAddress), + BuildMail(_inboxFolder.Id, DateTime.UtcNow.AddSeconds(-1), fromAddress: UnknownAddress), + BuildMail(_inboxFolder.Id, DateTime.UtcNow.AddSeconds(-2), fromAddress: _testAccount.Address) + }; + await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy)); + + var options = BuildOptions([_inboxFolder], createThreads: false, take: 10); + + // Act + var result = await _mailService.FetchMailsAsync(options); + + result.Should().HaveCount(3); + + // Known contact – resolved from AccountContact table + var knownResult = result.Single(m => m.FromAddress == KnownAddress); + knownResult.SenderContact!.Name.Should().Be("Known Sender"); + + // Unknown address – falls back to an ad-hoc contact built from From headers + var unknownResult = result.Single(m => m.FromAddress == UnknownAddress); + unknownResult.SenderContact!.Address.Should().Be(UnknownAddress); + + // Self-sent mail – contact built from account metadata, not the contact store + var selfResult = result.Single(m => m.FromAddress == _testAccount.Address); + selfResult.SenderContact!.Name.Should().Be(_testAccount.SenderName, + "self-sent mail must use account metadata for the sender contact"); + } + + // ── Performance: 1 000 mails / ~70 threads ───────────────────────────────── + + /// + /// Creates 1 000 mails: 70 threads of 7 mails each (490 mails) plus 510 standalone. + /// The mails are ordered newest-first in thread blocks so the default first-page + /// fetch (100 mails) naturally spans several complete threads and the tail of one + /// partial thread, letting us observe thread expansion. + /// + /// Two scenarios are measured and written to test output: + /// 1. First-page fetch (100 mails) plus automatic thread expansion. + /// 2. Full load of all 1 000 mails with threading enabled. + /// + /// A generous 5-second budget is asserted to catch catastrophic regressions + /// without being brittle on slow CI hardware. + /// + [Fact] + public async Task FetchMailsAsync_1000Mails_70Threads_CompletesWithinBudget() + { + // ── Arrange ──────────────────────────────────────────────────────────── + const int ThreadCount = 70; + const int MailsPerThread = 7; + const int TotalMails = 1_000; + const int StandaloneMails = TotalMails - (ThreadCount * MailsPerThread); // 510 + + // 40 rotating sender addresses; the first 20 have entries in the contact store. + var senders = Enumerable.Range(0, 40) + .Select(i => $"sender{i:D2}@example.com") + .ToList(); + + var knownContacts = senders.Take(20) + .Select((addr, i) => new AccountContact { Address = addr, Name = $"Sender {i}" }) + .ToList(); + await _databaseService.Connection.InsertAllAsync(knownContacts, typeof(AccountContact)); + + // Threads occupy the newest date slots (positions 0–489) so the default 100-mail + // page always intersects several threads, triggering sibling expansion. + var mails = new List(TotalMails); + var baseDate = DateTime.UtcNow; + int slot = 0; + + for (int t = 0; t < ThreadCount; t++) + { + var threadId = Guid.NewGuid().ToString(); + for (int m = 0; m < MailsPerThread; m++) + { + mails.Add(BuildMail( + _inboxFolder.Id, + baseDate.AddSeconds(-slot), + threadId: threadId, + fromAddress: senders[slot % senders.Count])); + slot++; + } + } + + for (int i = 0; i < StandaloneMails; i++) + { + mails.Add(BuildMail( + _inboxFolder.Id, + baseDate.AddSeconds(-slot), + fromAddress: senders[slot % senders.Count])); + slot++; + } + + await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy)); + + _output.WriteLine($"Inserted {TotalMails} mails — " + + $"{ThreadCount} threads × {MailsPerThread} mails + {StandaloneMails} standalone"); + _output.WriteLine(string.Empty); + + // ── Scenario 1: first page (default 100) + thread expansion ─────────── + // The 100 newest mails span threads 0–13 completely (14 × 7 = 98 mails) plus + // the first 2 mails of thread 14. Expansion must fetch thread 14's 5 siblings. + var optionsPage = BuildOptions([_inboxFolder], createThreads: true); + var sw = Stopwatch.StartNew(); + var pageResult = await _mailService.FetchMailsAsync(optionsPage); + sw.Stop(); + long pageMs = sw.ElapsedMilliseconds; + + _output.WriteLine("[Scenario 1 – first page + thread expansion]"); + _output.WriteLine($" Mails returned : {pageResult.Count} (expected > 100)"); + _output.WriteLine($" Elapsed : {pageMs} ms"); + _output.WriteLine(string.Empty); + + pageResult.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null); + + // Thread expansion must have added thread 14's 5 siblings beyond the 100-mail page. + pageResult.Count.Should().BeGreaterThan(100, + "thread expansion must pull in siblings that were beyond the initial 100-mail page"); + + // ── Scenario 2: full load of all 1 000 mails with threading ─────────── + var optionsAll = BuildOptions([_inboxFolder], createThreads: true, take: TotalMails); + sw.Restart(); + var allResult = await _mailService.FetchMailsAsync(optionsAll); + sw.Stop(); + long allMs = sw.ElapsedMilliseconds; + + _output.WriteLine($"[Scenario 2 – full load ({TotalMails} mails, threading enabled)]"); + _output.WriteLine($" Mails returned : {allResult.Count} (expected {TotalMails})"); + _output.WriteLine($" Elapsed : {allMs} ms"); + + allResult.Should().HaveCount(TotalMails, + "every mail must be returned when Take equals the total count"); + allResult.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null); + + // All 70 threads must be intact in the full result. + var threadGroups = allResult + .Where(m => !string.IsNullOrEmpty(m.ThreadId)) + .GroupBy(m => m.ThreadId!) + .ToList(); + + threadGroups.Should().HaveCount(ThreadCount, + "all 70 threads must be represented in the full load"); + threadGroups.Should().OnlyContain(g => g.Count() == MailsPerThread, + "every thread must contain exactly the expected number of mails"); + + allMs.Should().BeLessThan(5_000, + $"fetching {TotalMails} threaded mails via batched SQLite queries should complete well under 5 s"); + } + + // ── Helpers ──────────────────────────────────────────────────────────────── + + private static MailCopy BuildMail( + Guid folderId, + DateTime creationDate, + string? threadId = null, + string fromAddress = "external@example.com") + { + return new MailCopy + { + UniqueId = Guid.NewGuid(), + Id = Guid.NewGuid().ToString(), + FileId = Guid.NewGuid(), + FolderId = folderId, + Subject = $"Subject {Guid.NewGuid():N}", + PreviewText = "Preview text", + FromAddress = fromAddress, + FromName = fromAddress.Split('@')[0], + CreationDate = creationDate, + ThreadId = threadId, + IsRead = false + }; + } + + private static MailListInitializationOptions BuildOptions( + IEnumerable folders, + bool createThreads = true, + int take = 0) + { + return new MailListInitializationOptions( + Folders: folders, + FilterType: FilterOptionType.All, + SortingOptionType: SortingOptionType.ReceiveDate, + CreateThreads: createThreads, + IsFocusedOnly: null, + SearchQuery: null, + Take: take); + } + + /// + /// Builds a MailService wired to real FolderService, AccountService, and ContactService + /// all backed by the shared in-memory database, so the full SQL batch path is exercised. + /// + private static MailService BuildMailService(InMemoryDatabaseService db) + { + var signatureService = new Mock(); + var authProvider = new Mock(); + var mimeFileService = new Mock(); + var preferencesService = new Mock(); + + var accountService = new AccountService( + db, + signatureService.Object, + authProvider.Object, + mimeFileService.Object, + preferencesService.Object); + + var folderService = new FolderService(db, accountService); + var contactService = new ContactService(db); + + return new MailService( + db, + folderService, + contactService, + accountService, + signatureService.Object, + mimeFileService.Object, + preferencesService.Object); + } +} diff --git a/Wino.Services/ContactService.cs b/Wino.Services/ContactService.cs index 4c5aaa3c..7bb0094a 100644 --- a/Wino.Services/ContactService.cs +++ b/Wino.Services/ContactService.cs @@ -37,6 +37,18 @@ public class ContactService : BaseDatabaseService, IContactService public Task GetAddressInformationByAddressAsync(string address) => Connection.Table().FirstOrDefaultAsync(a => a.Address == address); + public Task> GetContactsByAddressesAsync(IEnumerable addresses) + { + var addressList = addresses?.Where(a => !string.IsNullOrEmpty(a)).Distinct().ToList(); + if (addressList == null || addressList.Count == 0) + return Task.FromResult(new List()); + + var placeholders = string.Join(",", addressList.Select(_ => "?")); + return Connection.QueryAsync( + $"SELECT * FROM AccountContact WHERE Address IN ({placeholders})", + addressList.Cast().ToArray()); + } + public async Task SaveAddressInformationAsync(MimeMessage message) { if (message == null) return; diff --git a/Wino.Services/MailService.cs b/Wino.Services/MailService.cs index 987f0d4d..6dd16d77 100644 --- a/Wino.Services/MailService.cs +++ b/Wino.Services/MailService.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text; @@ -324,91 +323,163 @@ public class MailService : BaseDatabaseService, IMailService public async Task> FetchMailsAsync(MailListInitializationOptions options, CancellationToken cancellationToken = default) { - List mails = null; + List mails; - // If user performs an online search, mail copies are passed to options. if (options.PreFetchMailCopies != null) { mails = ApplyOptionsToPreFetchedMails(options); } else { - // If not just do the query. var (query, parameters) = BuildMailFetchQuery(options); mails = await Connection.QueryAsync(query, parameters); } - ConcurrentDictionary folderCache = new(); - ConcurrentDictionary accountCache = new(); - ConcurrentDictionary contactCache = new(); - - // Populate Folder Assignment for each single mail, to be able later group by "MailAccountId". - // This is needed to execute threading strategy by account type. - // Avoid DBs calls as possible, storing info in a dictionary. - foreach (var mail in mails) - { - await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false); - } - - // Remove items that has no assigned account or folder. - mails.RemoveAll(a => a.AssignedAccount == null || a.AssignedFolder == null); + if (mails.Count == 0) + return mails; cancellationToken.ThrowIfCancellationRequested(); - // If CreateThreads is false, just return the mails as-is - if (!options.CreateThreads) + // Pre-load all data needed for property assignment in as few DB round-trips as possible. + // 1. Seed the folder cache directly from the options folders - these cover the vast majority + // of mails in a normal folder view and require zero extra DB calls. + var folderCache = options.Folders + .OfType() + .ToDictionary(f => f.Id); + + // 2. Load all accounts in one call (typically 1-5 accounts) instead of N per-mail lookups. + var allAccounts = await _accountService.GetAccountsAsync().ConfigureAwait(false); + var accountCache = allAccounts.ToDictionary(a => a.Id); + + // 3. Fetch any folders not already in the cache (rare for normal views, common for merged inboxes + // that include Sent/Draft copies belonging to different folder objects). + var uncachedFolderIds = mails + .Select(m => m.FolderId) + .Distinct() + .Where(id => !folderCache.ContainsKey(id)) + .ToList(); + + if (uncachedFolderIds.Count > 0) { - return [.. mails]; + var folders = await Task.WhenAll( + uncachedFolderIds.Select(id => _folderService.GetFolderAsync(id))).ConfigureAwait(false); + + foreach (var f in folders.Where(f => f != null)) + folderCache[f.Id] = f; } - // Include other mails in the same threads - batch process to reduce DB calls - var expandedMails = new List(mails); + // 4. Batch-fetch all sender contacts in a single SQL IN(...) query instead of one query per mail. + var uniqueAddresses = mails + .Where(m => !string.IsNullOrEmpty(m.FromAddress)) + .Select(m => m.FromAddress) + .Distinct() + .ToList(); + + var contactList = await _contactService.GetContactsByAddressesAsync(uniqueAddresses).ConfigureAwait(false); + var contactCache = contactList.ToDictionary(c => c.Address); + + cancellationToken.ThrowIfCancellationRequested(); + + // 5. Assign all properties synchronously from the pre-loaded in-memory caches - no DB calls here. + AssignPropertiesFromCaches(mails, folderCache, accountCache, contactCache); + mails.RemoveAll(m => m.AssignedAccount == null || m.AssignedFolder == null); + + if (!options.CreateThreads || mails.Count == 0) + return [.. mails]; + + // 6. Expand threads: one batch query for all sibling mails across all threads. var uniqueThreadIds = mails .Where(m => !string.IsNullOrEmpty(m.ThreadId)) .Select(m => m.ThreadId) .Distinct() .ToList(); - if (uniqueThreadIds.Count > 0) + if (uniqueThreadIds.Count == 0) + return [.. mails]; + + var existingMailIds = mails.Select(m => m.Id).ToHashSet(); + var threadMails = await GetMailsByThreadIdsAsync(uniqueThreadIds, existingMailIds).ConfigureAwait(false); + + if (threadMails?.Count > 0) { - // Get all thread mails in a single DB call - var existingMailIds = expandedMails.Select(m => m.Id).ToHashSet(); - var allThreadMails = await GetMailsByThreadIdsAsync(uniqueThreadIds, existingMailIds).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); - if (allThreadMails?.Count > 0) + // Load any folders that thread mails belong to but are not yet cached. + var newFolderIds = threadMails + .Select(m => m.FolderId) + .Distinct() + .Where(id => !folderCache.ContainsKey(id)) + .ToList(); + + if (newFolderIds.Count > 0) { - // Process thread mails in parallel to improve performance - var tasks = allThreadMails.Select(async threadMail => - { - await LoadAssignedPropertiesWithCacheAsync(threadMail, folderCache, accountCache, contactCache).ConfigureAwait(false); - return threadMail; - }); + var newFolders = await Task.WhenAll( + newFolderIds.Select(id => _folderService.GetFolderAsync(id))).ConfigureAwait(false); - var processedThreadMails = await Task.WhenAll(tasks).ConfigureAwait(false); - - // Filter out items with no assigned account or folder - var validThreadMails = processedThreadMails.Where(m => m.AssignedAccount != null && m.AssignedFolder != null); - - expandedMails.AddRange(validThreadMails); + foreach (var f in newFolders.Where(f => f != null)) + folderCache[f.Id] = f; } - cancellationToken.ThrowIfCancellationRequested(); + // Batch-fetch contacts for any new senders in thread mails. + var newAddresses = threadMails + .Where(m => !string.IsNullOrEmpty(m.FromAddress) && !contactCache.ContainsKey(m.FromAddress)) + .Select(m => m.FromAddress) + .Distinct() + .ToList(); + + if (newAddresses.Count > 0) + { + var newContacts = await _contactService.GetContactsByAddressesAsync(newAddresses).ConfigureAwait(false); + foreach (var c in newContacts.Where(c => c != null)) + contactCache[c.Address] = c; + } + + AssignPropertiesFromCaches(threadMails, folderCache, accountCache, contactCache); + mails.AddRange(threadMails.Where(m => m.AssignedAccount != null && m.AssignedFolder != null)); } - return [.. expandedMails]; + cancellationToken.ThrowIfCancellationRequested(); + return [.. mails]; } - private async Task> GetMailsByThreadIdAsync(string threadId, HashSet excludeMailIds) + /// + /// Assigns AssignedFolder, AssignedAccount, and SenderContact to each mail from pre-loaded + /// in-memory dictionaries. No DB calls are made here. + /// + private void AssignPropertiesFromCaches( + List mails, + Dictionary folderCache, + Dictionary accountCache, + Dictionary contactCache) { - if (string.IsNullOrEmpty(threadId)) - return []; + foreach (var mail in mails) + { + if (!folderCache.TryGetValue(mail.FolderId, out var folder)) + continue; - var placeholders = string.Join(",", excludeMailIds.Select(_ => "?")); - var sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId = ? AND Id NOT IN ({placeholders})"; - var parameters = new List { threadId }; - parameters.AddRange(excludeMailIds.Cast()); + if (!accountCache.TryGetValue(folder.MailAccountId, out var account)) + continue; - return await Connection.QueryAsync(sql, parameters.ToArray()); + mail.AssignedFolder = folder; + mail.AssignedAccount = account; + + // Self-sent mails (e.g. Sent folder): construct contact from account meta + // to get the up-to-date profile picture without a DB roundtrip. + if (!string.IsNullOrEmpty(mail.FromAddress) && mail.FromAddress == account.Address) + { + mail.SenderContact = new AccountContact + { + Address = account.Address, + Name = account.SenderName, + Base64ContactPicture = account.Base64ProfilePictureData + }; + } + else + { + contactCache.TryGetValue(mail.FromAddress ?? string.Empty, out var contact); + mail.SenderContact = contact ?? CreateUnknownContact(mail.FromName, mail.FromAddress); + } + } } private async Task> GetMailsByThreadIdsAsync(List threadIds, HashSet excludeMailIds) @@ -417,63 +488,24 @@ public class MailService : BaseDatabaseService, IMailService return []; var threadPlaceholders = string.Join(",", threadIds.Select(_ => "?")); - var excludePlaceholders = string.Join(",", excludeMailIds.Select(_ => "?")); - var sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders}) AND Id NOT IN ({excludePlaceholders})"; var parameters = new List(); parameters.AddRange(threadIds.Cast()); - parameters.AddRange(excludeMailIds.Cast()); + + string sql; + if (excludeMailIds.Count > 0) + { + var excludePlaceholders = string.Join(",", excludeMailIds.Select(_ => "?")); + sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders}) AND Id NOT IN ({excludePlaceholders})"; + parameters.AddRange(excludeMailIds.Cast()); + } + else + { + sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders})"; + } return await Connection.QueryAsync(sql, parameters.ToArray()).ConfigureAwait(false); } - /// - /// This method should used for operations with multiple mailItems. Don't use this for single mail items. - /// Called method should provide own instances for caches. - /// - private async Task LoadAssignedPropertiesWithCacheAsync(MailCopy mail, ConcurrentDictionary folderCache, ConcurrentDictionary accountCache, ConcurrentDictionary contactCache) - { - if (mail is MailCopy mailCopy) - { - var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment); - MailAccount accountAssignment = null; - if (!isFolderCached) - { - folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false); - folderCache.TryAdd(mailCopy.FolderId, folderAssignment); - } - - if (folderAssignment != null) - { - var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment); - if (!isAccountCached) - { - accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false); - - accountCache.TryAdd(folderAssignment.MailAccountId, accountAssignment); - } - } - - AccountContact contactAssignment = null; - - bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) && - contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment); - - if (!isContactCached && accountAssignment != null) - { - contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false); - - if (contactAssignment != null) - { - contactCache.TryAdd(mailCopy.FromAddress, contactAssignment); - } - } - - mailCopy.AssignedFolder = folderAssignment; - mailCopy.AssignedAccount = accountAssignment; - mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress); - } - } - private static AccountContact CreateUnknownContact(string fromName, string fromAddress) { if (string.IsNullOrEmpty(fromName) && string.IsNullOrEmpty(fromAddress)) @@ -1407,14 +1439,19 @@ public class MailService : BaseDatabaseService, IMailService var mailCopies = await Connection.QueryAsync(sql, mailCopyIds.Cast().ToArray()); if (mailCopies?.Count == 0) return []; - ConcurrentDictionary folderCache = new(); - ConcurrentDictionary accountCache = new(); - ConcurrentDictionary contactCache = new(); + var folderIds = mailCopies.Select(m => m.FolderId).Distinct().ToList(); + var folderTasks = folderIds.Select(id => _folderService.GetFolderAsync(id)); + var folders = await Task.WhenAll(folderTasks).ConfigureAwait(false); + var folderCache = folders.Where(f => f != null).ToDictionary(f => f.Id); - foreach (var mail in mailCopies) - { - await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false); - } + var allAccounts = await _accountService.GetAccountsAsync().ConfigureAwait(false); + var accountCache = allAccounts.ToDictionary(a => a.Id); + + var addresses = mailCopies.Where(m => !string.IsNullOrEmpty(m.FromAddress)).Select(m => m.FromAddress).Distinct().ToList(); + var contactList = await _contactService.GetContactsByAddressesAsync(addresses).ConfigureAwait(false); + var contactCache = contactList.ToDictionary(c => c.Address); + + AssignPropertiesFromCaches(mailCopies, folderCache, accountCache, contactCache); return mailCopies; }