Optimize mail fetching with batch DB queries and in-memory caching (#827)
* perf: batch-load folders, accounts, and contacts in FetchMailsAsync Replace the sequential per-mail property-loading loop with a three-step batch pre-load strategy, eliminating the N+1 DB call pattern that was the main bottleneck when building the mail list with threading enabled. Changes: - Pre-seed the folder cache from MailListInitializationOptions.Folders so that the most common folders (inbox, sent, etc.) never trigger a DB lookup at all. - Load all accounts in a single GetAccountsAsync() call instead of one GetAccountAsync() call per mail (typically 1–5 accounts total). - Fetch all sender contacts in a single SQL IN(...) query via the new GetContactsByAddressesAsync() method instead of one query per address. - Property assignment is now fully synchronous (no awaits in the loop) since all data is pre-loaded into plain Dictionary<K,V>. - Thread-expansion follows the same pattern: new folder IDs are loaded in parallel via Task.WhenAll; new contact addresses are batch-fetched with a second IN(...) query. - Also apply batch pre-loading to GetMailItemsAsync (used by merge-inbox sync path) which had the same sequential issue. - Remove the now-unused LoadAssignedPropertiesWithCacheAsync helper and the ConcurrentDictionary dependency it required. - Tighten GetMailsByThreadIdsAsync to skip the Id NOT IN clause entirely when the exclusion set is empty. https://claude.ai/code/session_018bqahGc6zi95JJhc2aARKS * test: add MailFetchingTests with correctness and performance coverage Adds integration tests for MailService.FetchMailsAsync that exercise the full real-service stack (MailService → FolderService / AccountService / ContactService) backed by the shared in-memory SQLite helper. Four tests are included: • ExpandsSiblingsOutsidePage – proves thread expansion fetches mails that fall beyond the initial SQL page (6 mails, page=4, expects 6 returned). • NeverExpandsSiblings – proves threading is truly opt-in; with CreateThreads=false the result exactly matches the raw page size. • ResolvesFromAllThreeSources – verifies contact resolution for a known contact (from the AccountContact table), an unknown sender (ad-hoc fallback), and a self-sent mail (built from account metadata). • 1000Mails_70Threads_CompletesWithinBudget – the performance scenario: 1 000 mails (70 threads × 7 + 510 standalone), 40 rotating sender addresses (20 with DB contacts). Times and reports two scenarios: - Default first-page fetch (100 mails) + expansion of one partial thread (expects > 100 mails returned). - Full load of all 1 000 mails with threading enabled (expects exactly 1 000 mails returned, all 70 threads intact, < 5 s). Elapsed times for both scenarios are written to xUnit test output so they appear in CI logs and can be tracked across builds. https://claude.ai/code/session_018bqahGc6zi95JJhc2aARKS --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Burak Kaan Köse
@@ -11,6 +11,7 @@ public interface IContactService
|
||||
{
|
||||
Task<List<AccountContact>> GetAddressInformationAsync(string queryText);
|
||||
Task<AccountContact> GetAddressInformationByAddressAsync(string address);
|
||||
Task<List<AccountContact>> GetContactsByAddressesAsync(IEnumerable<string> addresses);
|
||||
Task SaveAddressInformationAsync(MimeMessage message);
|
||||
Task SaveAddressInformationAsync(IEnumerable<AccountContact> contacts);
|
||||
Task<AccountContact> CreateNewContactAsync(string address, string displayName);
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
using System.Diagnostics;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
using Wino.Core.Domain.Entities.Mail;
|
||||
using Wino.Core.Domain.Entities.Shared;
|
||||
using Wino.Core.Domain.Enums;
|
||||
using Wino.Core.Domain.Interfaces;
|
||||
using Wino.Core.Domain.Models.MailItem;
|
||||
using Wino.Core.Tests.Helpers;
|
||||
using Wino.Services;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace Wino.Core.Tests.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Integration tests for MailService.FetchMailsAsync that verify the correctness of
|
||||
/// thread expansion and contact resolution, and track performance for large inboxes.
|
||||
///
|
||||
/// All tests run against a real in-memory SQLite file via the full service stack
|
||||
/// (MailService → FolderService / AccountService / ContactService) so that the
|
||||
/// batch-query path introduced in the performance optimisation is exercised end-to-end.
|
||||
/// </summary>
|
||||
public class MailFetchingTests : IAsyncLifetime
|
||||
{
|
||||
// ── Infrastructure ─────────────────────────────────────────────────────────
|
||||
|
||||
private readonly ITestOutputHelper _output;
|
||||
private InMemoryDatabaseService _databaseService = null!;
|
||||
private MailService _mailService = null!;
|
||||
private MailAccount _testAccount = null!;
|
||||
private MailItemFolder _inboxFolder = null!;
|
||||
|
||||
public MailFetchingTests(ITestOutputHelper output)
|
||||
{
|
||||
_output = output;
|
||||
}
|
||||
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_databaseService = new InMemoryDatabaseService();
|
||||
await _databaseService.InitializeAsync();
|
||||
|
||||
_testAccount = new MailAccount
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Name = "Test Account",
|
||||
Address = "me@test.local",
|
||||
SenderName = "Test User",
|
||||
ProviderType = MailProviderType.IMAP4
|
||||
};
|
||||
|
||||
_inboxFolder = new MailItemFolder
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
MailAccountId = _testAccount.Id,
|
||||
FolderName = "Inbox",
|
||||
SpecialFolderType = SpecialFolderType.Inbox,
|
||||
IsSystemFolder = true,
|
||||
IsSynchronizationEnabled = true
|
||||
};
|
||||
|
||||
await _databaseService.Connection.InsertAsync(_testAccount, typeof(MailAccount));
|
||||
await _databaseService.Connection.InsertAsync(_inboxFolder, typeof(MailItemFolder));
|
||||
|
||||
_mailService = BuildMailService(_databaseService);
|
||||
}
|
||||
|
||||
public async Task DisposeAsync() => await _databaseService.DisposeAsync();
|
||||
|
||||
// ── Correctness: threading ON ──────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that thread siblings which fall outside the initial SQL page are
|
||||
/// fetched by the expansion step, so every thread is always fully represented.
|
||||
///
|
||||
/// Setup: 2 threads of 3 mails each (6 mails total), page size = 4.
|
||||
/// The main query retrieves Thread A (3 mails, newest) and Thread B mail 1 (position 4).
|
||||
/// Thread expansion must then fetch Thread B mails 2-3 that were beyond the page.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task FetchMailsAsync_WithThreadingEnabled_ExpandsSiblingsOutsidePage()
|
||||
{
|
||||
const int PageSize = 4;
|
||||
var threadA = Guid.NewGuid().ToString();
|
||||
var threadB = Guid.NewGuid().ToString();
|
||||
var baseDate = DateTime.UtcNow;
|
||||
|
||||
var mails = new List<MailCopy>
|
||||
{
|
||||
// Thread A – all 3 land within the first page (positions 1–3)
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-1), threadId: threadA),
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-2), threadId: threadA),
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-3), threadId: threadA),
|
||||
// Thread B – only position 4 lands in the page; 5 and 6 must be expanded
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-4), threadId: threadB),
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-5), threadId: threadB),
|
||||
BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-6), threadId: threadB)
|
||||
};
|
||||
await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy));
|
||||
|
||||
var options = BuildOptions([_inboxFolder], createThreads: true, take: PageSize);
|
||||
|
||||
// Act
|
||||
var result = await _mailService.FetchMailsAsync(options);
|
||||
|
||||
// Assert – all 6 mails returned even though the page only held 4
|
||||
result.Should().HaveCount(6,
|
||||
"the 2 Thread B siblings outside the initial page must be fetched by expansion");
|
||||
result.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null,
|
||||
"every returned mail must have its account and folder resolved");
|
||||
result.Count(m => m.ThreadId == threadA).Should().Be(3, "Thread A must be complete");
|
||||
result.Count(m => m.ThreadId == threadB).Should().Be(3, "Thread B must be complete");
|
||||
}
|
||||
|
||||
// ── Correctness: threading OFF ─────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that when threading is disabled the result exactly matches the raw
|
||||
/// SQL page — no sibling expansion occurs.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task FetchMailsAsync_WithThreadingDisabled_NeverExpandsSiblings()
|
||||
{
|
||||
const int PageSize = 4;
|
||||
var threadId = Guid.NewGuid().ToString();
|
||||
var baseDate = DateTime.UtcNow;
|
||||
|
||||
// 6 mails all sharing a ThreadId; with threading OFF only the first 4 come back
|
||||
var mails = Enumerable.Range(0, 6)
|
||||
.Select(i => BuildMail(_inboxFolder.Id, baseDate.AddSeconds(-i), threadId: threadId))
|
||||
.ToList();
|
||||
|
||||
await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy));
|
||||
|
||||
var options = BuildOptions([_inboxFolder], createThreads: false, take: PageSize);
|
||||
|
||||
// Act
|
||||
var result = await _mailService.FetchMailsAsync(options);
|
||||
|
||||
// Assert – exactly the page size; no expansion happened
|
||||
result.Should().HaveCount(PageSize,
|
||||
"with threading disabled the result must match the raw page size");
|
||||
result.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null);
|
||||
}
|
||||
|
||||
// ── Correctness: contact resolution ───────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that sender contacts are resolved from three distinct paths:
|
||||
/// the contact store (known sender), the unknown-sender fallback, and the
|
||||
/// account-metadata shortcut used for self-sent mails.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task FetchMailsAsync_SenderContact_ResolvesFromAllThreeSources()
|
||||
{
|
||||
const string KnownAddress = "known@example.com";
|
||||
const string UnknownAddress = "unknown@example.com";
|
||||
|
||||
await _databaseService.Connection.InsertAsync(
|
||||
new AccountContact { Address = KnownAddress, Name = "Known Sender" },
|
||||
typeof(AccountContact));
|
||||
|
||||
var mails = new List<MailCopy>
|
||||
{
|
||||
BuildMail(_inboxFolder.Id, DateTime.UtcNow, fromAddress: KnownAddress),
|
||||
BuildMail(_inboxFolder.Id, DateTime.UtcNow.AddSeconds(-1), fromAddress: UnknownAddress),
|
||||
BuildMail(_inboxFolder.Id, DateTime.UtcNow.AddSeconds(-2), fromAddress: _testAccount.Address)
|
||||
};
|
||||
await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy));
|
||||
|
||||
var options = BuildOptions([_inboxFolder], createThreads: false, take: 10);
|
||||
|
||||
// Act
|
||||
var result = await _mailService.FetchMailsAsync(options);
|
||||
|
||||
result.Should().HaveCount(3);
|
||||
|
||||
// Known contact – resolved from AccountContact table
|
||||
var knownResult = result.Single(m => m.FromAddress == KnownAddress);
|
||||
knownResult.SenderContact!.Name.Should().Be("Known Sender");
|
||||
|
||||
// Unknown address – falls back to an ad-hoc contact built from From headers
|
||||
var unknownResult = result.Single(m => m.FromAddress == UnknownAddress);
|
||||
unknownResult.SenderContact!.Address.Should().Be(UnknownAddress);
|
||||
|
||||
// Self-sent mail – contact built from account metadata, not the contact store
|
||||
var selfResult = result.Single(m => m.FromAddress == _testAccount.Address);
|
||||
selfResult.SenderContact!.Name.Should().Be(_testAccount.SenderName,
|
||||
"self-sent mail must use account metadata for the sender contact");
|
||||
}
|
||||
|
||||
// ── Performance: 1 000 mails / ~70 threads ─────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Creates 1 000 mails: 70 threads of 7 mails each (490 mails) plus 510 standalone.
|
||||
/// The mails are ordered newest-first in thread blocks so the default first-page
|
||||
/// fetch (100 mails) naturally spans several complete threads and the tail of one
|
||||
/// partial thread, letting us observe thread expansion.
|
||||
///
|
||||
/// Two scenarios are measured and written to test output:
|
||||
/// 1. First-page fetch (100 mails) plus automatic thread expansion.
|
||||
/// 2. Full load of all 1 000 mails with threading enabled.
|
||||
///
|
||||
/// A generous 5-second budget is asserted to catch catastrophic regressions
|
||||
/// without being brittle on slow CI hardware.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task FetchMailsAsync_1000Mails_70Threads_CompletesWithinBudget()
|
||||
{
|
||||
// ── Arrange ────────────────────────────────────────────────────────────
|
||||
const int ThreadCount = 70;
|
||||
const int MailsPerThread = 7;
|
||||
const int TotalMails = 1_000;
|
||||
const int StandaloneMails = TotalMails - (ThreadCount * MailsPerThread); // 510
|
||||
|
||||
// 40 rotating sender addresses; the first 20 have entries in the contact store.
|
||||
var senders = Enumerable.Range(0, 40)
|
||||
.Select(i => $"sender{i:D2}@example.com")
|
||||
.ToList();
|
||||
|
||||
var knownContacts = senders.Take(20)
|
||||
.Select((addr, i) => new AccountContact { Address = addr, Name = $"Sender {i}" })
|
||||
.ToList();
|
||||
await _databaseService.Connection.InsertAllAsync(knownContacts, typeof(AccountContact));
|
||||
|
||||
// Threads occupy the newest date slots (positions 0–489) so the default 100-mail
|
||||
// page always intersects several threads, triggering sibling expansion.
|
||||
var mails = new List<MailCopy>(TotalMails);
|
||||
var baseDate = DateTime.UtcNow;
|
||||
int slot = 0;
|
||||
|
||||
for (int t = 0; t < ThreadCount; t++)
|
||||
{
|
||||
var threadId = Guid.NewGuid().ToString();
|
||||
for (int m = 0; m < MailsPerThread; m++)
|
||||
{
|
||||
mails.Add(BuildMail(
|
||||
_inboxFolder.Id,
|
||||
baseDate.AddSeconds(-slot),
|
||||
threadId: threadId,
|
||||
fromAddress: senders[slot % senders.Count]));
|
||||
slot++;
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; i < StandaloneMails; i++)
|
||||
{
|
||||
mails.Add(BuildMail(
|
||||
_inboxFolder.Id,
|
||||
baseDate.AddSeconds(-slot),
|
||||
fromAddress: senders[slot % senders.Count]));
|
||||
slot++;
|
||||
}
|
||||
|
||||
await _databaseService.Connection.InsertAllAsync(mails, typeof(MailCopy));
|
||||
|
||||
_output.WriteLine($"Inserted {TotalMails} mails — " +
|
||||
$"{ThreadCount} threads × {MailsPerThread} mails + {StandaloneMails} standalone");
|
||||
_output.WriteLine(string.Empty);
|
||||
|
||||
// ── Scenario 1: first page (default 100) + thread expansion ───────────
|
||||
// The 100 newest mails span threads 0–13 completely (14 × 7 = 98 mails) plus
|
||||
// the first 2 mails of thread 14. Expansion must fetch thread 14's 5 siblings.
|
||||
var optionsPage = BuildOptions([_inboxFolder], createThreads: true);
|
||||
var sw = Stopwatch.StartNew();
|
||||
var pageResult = await _mailService.FetchMailsAsync(optionsPage);
|
||||
sw.Stop();
|
||||
long pageMs = sw.ElapsedMilliseconds;
|
||||
|
||||
_output.WriteLine("[Scenario 1 – first page + thread expansion]");
|
||||
_output.WriteLine($" Mails returned : {pageResult.Count} (expected > 100)");
|
||||
_output.WriteLine($" Elapsed : {pageMs} ms");
|
||||
_output.WriteLine(string.Empty);
|
||||
|
||||
pageResult.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null);
|
||||
|
||||
// Thread expansion must have added thread 14's 5 siblings beyond the 100-mail page.
|
||||
pageResult.Count.Should().BeGreaterThan(100,
|
||||
"thread expansion must pull in siblings that were beyond the initial 100-mail page");
|
||||
|
||||
// ── Scenario 2: full load of all 1 000 mails with threading ───────────
|
||||
var optionsAll = BuildOptions([_inboxFolder], createThreads: true, take: TotalMails);
|
||||
sw.Restart();
|
||||
var allResult = await _mailService.FetchMailsAsync(optionsAll);
|
||||
sw.Stop();
|
||||
long allMs = sw.ElapsedMilliseconds;
|
||||
|
||||
_output.WriteLine($"[Scenario 2 – full load ({TotalMails} mails, threading enabled)]");
|
||||
_output.WriteLine($" Mails returned : {allResult.Count} (expected {TotalMails})");
|
||||
_output.WriteLine($" Elapsed : {allMs} ms");
|
||||
|
||||
allResult.Should().HaveCount(TotalMails,
|
||||
"every mail must be returned when Take equals the total count");
|
||||
allResult.Should().OnlyContain(m => m.AssignedAccount != null && m.AssignedFolder != null);
|
||||
|
||||
// All 70 threads must be intact in the full result.
|
||||
var threadGroups = allResult
|
||||
.Where(m => !string.IsNullOrEmpty(m.ThreadId))
|
||||
.GroupBy(m => m.ThreadId!)
|
||||
.ToList();
|
||||
|
||||
threadGroups.Should().HaveCount(ThreadCount,
|
||||
"all 70 threads must be represented in the full load");
|
||||
threadGroups.Should().OnlyContain(g => g.Count() == MailsPerThread,
|
||||
"every thread must contain exactly the expected number of mails");
|
||||
|
||||
allMs.Should().BeLessThan(5_000,
|
||||
$"fetching {TotalMails} threaded mails via batched SQLite queries should complete well under 5 s");
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
private static MailCopy BuildMail(
|
||||
Guid folderId,
|
||||
DateTime creationDate,
|
||||
string? threadId = null,
|
||||
string fromAddress = "external@example.com")
|
||||
{
|
||||
return new MailCopy
|
||||
{
|
||||
UniqueId = Guid.NewGuid(),
|
||||
Id = Guid.NewGuid().ToString(),
|
||||
FileId = Guid.NewGuid(),
|
||||
FolderId = folderId,
|
||||
Subject = $"Subject {Guid.NewGuid():N}",
|
||||
PreviewText = "Preview text",
|
||||
FromAddress = fromAddress,
|
||||
FromName = fromAddress.Split('@')[0],
|
||||
CreationDate = creationDate,
|
||||
ThreadId = threadId,
|
||||
IsRead = false
|
||||
};
|
||||
}
|
||||
|
||||
private static MailListInitializationOptions BuildOptions(
|
||||
IEnumerable<MailItemFolder> folders,
|
||||
bool createThreads = true,
|
||||
int take = 0)
|
||||
{
|
||||
return new MailListInitializationOptions(
|
||||
Folders: folders,
|
||||
FilterType: FilterOptionType.All,
|
||||
SortingOptionType: SortingOptionType.ReceiveDate,
|
||||
CreateThreads: createThreads,
|
||||
IsFocusedOnly: null,
|
||||
SearchQuery: null,
|
||||
Take: take);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a MailService wired to real FolderService, AccountService, and ContactService
|
||||
/// all backed by the shared in-memory database, so the full SQL batch path is exercised.
|
||||
/// </summary>
|
||||
private static MailService BuildMailService(InMemoryDatabaseService db)
|
||||
{
|
||||
var signatureService = new Mock<ISignatureService>();
|
||||
var authProvider = new Mock<IAuthenticationProvider>();
|
||||
var mimeFileService = new Mock<IMimeFileService>();
|
||||
var preferencesService = new Mock<IPreferencesService>();
|
||||
|
||||
var accountService = new AccountService(
|
||||
db,
|
||||
signatureService.Object,
|
||||
authProvider.Object,
|
||||
mimeFileService.Object,
|
||||
preferencesService.Object);
|
||||
|
||||
var folderService = new FolderService(db, accountService);
|
||||
var contactService = new ContactService(db);
|
||||
|
||||
return new MailService(
|
||||
db,
|
||||
folderService,
|
||||
contactService,
|
||||
accountService,
|
||||
signatureService.Object,
|
||||
mimeFileService.Object,
|
||||
preferencesService.Object);
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,18 @@ public class ContactService : BaseDatabaseService, IContactService
|
||||
public Task<AccountContact> GetAddressInformationByAddressAsync(string address)
|
||||
=> Connection.Table<AccountContact>().FirstOrDefaultAsync(a => a.Address == address);
|
||||
|
||||
public Task<List<AccountContact>> GetContactsByAddressesAsync(IEnumerable<string> addresses)
|
||||
{
|
||||
var addressList = addresses?.Where(a => !string.IsNullOrEmpty(a)).Distinct().ToList();
|
||||
if (addressList == null || addressList.Count == 0)
|
||||
return Task.FromResult(new List<AccountContact>());
|
||||
|
||||
var placeholders = string.Join(",", addressList.Select(_ => "?"));
|
||||
return Connection.QueryAsync<AccountContact>(
|
||||
$"SELECT * FROM AccountContact WHERE Address IN ({placeholders})",
|
||||
addressList.Cast<object>().ToArray());
|
||||
}
|
||||
|
||||
public async Task SaveAddressInformationAsync(MimeMessage message)
|
||||
{
|
||||
if (message == null) return;
|
||||
|
||||
+145
-108
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
@@ -324,91 +323,163 @@ public class MailService : BaseDatabaseService, IMailService
|
||||
|
||||
public async Task<List<MailCopy>> FetchMailsAsync(MailListInitializationOptions options, CancellationToken cancellationToken = default)
|
||||
{
|
||||
List<MailCopy> mails = null;
|
||||
List<MailCopy> mails;
|
||||
|
||||
// If user performs an online search, mail copies are passed to options.
|
||||
if (options.PreFetchMailCopies != null)
|
||||
{
|
||||
mails = ApplyOptionsToPreFetchedMails(options);
|
||||
}
|
||||
else
|
||||
{
|
||||
// If not just do the query.
|
||||
var (query, parameters) = BuildMailFetchQuery(options);
|
||||
mails = await Connection.QueryAsync<MailCopy>(query, parameters);
|
||||
}
|
||||
|
||||
ConcurrentDictionary<Guid, MailItemFolder> folderCache = new();
|
||||
ConcurrentDictionary<Guid, MailAccount> accountCache = new();
|
||||
ConcurrentDictionary<string, AccountContact> contactCache = new();
|
||||
|
||||
// Populate Folder Assignment for each single mail, to be able later group by "MailAccountId".
|
||||
// This is needed to execute threading strategy by account type.
|
||||
// Avoid DBs calls as possible, storing info in a dictionary.
|
||||
foreach (var mail in mails)
|
||||
{
|
||||
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Remove items that has no assigned account or folder.
|
||||
mails.RemoveAll(a => a.AssignedAccount == null || a.AssignedFolder == null);
|
||||
if (mails.Count == 0)
|
||||
return mails;
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
// If CreateThreads is false, just return the mails as-is
|
||||
if (!options.CreateThreads)
|
||||
// Pre-load all data needed for property assignment in as few DB round-trips as possible.
|
||||
// 1. Seed the folder cache directly from the options folders - these cover the vast majority
|
||||
// of mails in a normal folder view and require zero extra DB calls.
|
||||
var folderCache = options.Folders
|
||||
.OfType<MailItemFolder>()
|
||||
.ToDictionary(f => f.Id);
|
||||
|
||||
// 2. Load all accounts in one call (typically 1-5 accounts) instead of N per-mail lookups.
|
||||
var allAccounts = await _accountService.GetAccountsAsync().ConfigureAwait(false);
|
||||
var accountCache = allAccounts.ToDictionary(a => a.Id);
|
||||
|
||||
// 3. Fetch any folders not already in the cache (rare for normal views, common for merged inboxes
|
||||
// that include Sent/Draft copies belonging to different folder objects).
|
||||
var uncachedFolderIds = mails
|
||||
.Select(m => m.FolderId)
|
||||
.Distinct()
|
||||
.Where(id => !folderCache.ContainsKey(id))
|
||||
.ToList();
|
||||
|
||||
if (uncachedFolderIds.Count > 0)
|
||||
{
|
||||
return [.. mails];
|
||||
var folders = await Task.WhenAll(
|
||||
uncachedFolderIds.Select(id => _folderService.GetFolderAsync(id))).ConfigureAwait(false);
|
||||
|
||||
foreach (var f in folders.Where(f => f != null))
|
||||
folderCache[f.Id] = f;
|
||||
}
|
||||
|
||||
// Include other mails in the same threads - batch process to reduce DB calls
|
||||
var expandedMails = new List<MailCopy>(mails);
|
||||
// 4. Batch-fetch all sender contacts in a single SQL IN(...) query instead of one query per mail.
|
||||
var uniqueAddresses = mails
|
||||
.Where(m => !string.IsNullOrEmpty(m.FromAddress))
|
||||
.Select(m => m.FromAddress)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
var contactList = await _contactService.GetContactsByAddressesAsync(uniqueAddresses).ConfigureAwait(false);
|
||||
var contactCache = contactList.ToDictionary(c => c.Address);
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
// 5. Assign all properties synchronously from the pre-loaded in-memory caches - no DB calls here.
|
||||
AssignPropertiesFromCaches(mails, folderCache, accountCache, contactCache);
|
||||
mails.RemoveAll(m => m.AssignedAccount == null || m.AssignedFolder == null);
|
||||
|
||||
if (!options.CreateThreads || mails.Count == 0)
|
||||
return [.. mails];
|
||||
|
||||
// 6. Expand threads: one batch query for all sibling mails across all threads.
|
||||
var uniqueThreadIds = mails
|
||||
.Where(m => !string.IsNullOrEmpty(m.ThreadId))
|
||||
.Select(m => m.ThreadId)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
if (uniqueThreadIds.Count > 0)
|
||||
if (uniqueThreadIds.Count == 0)
|
||||
return [.. mails];
|
||||
|
||||
var existingMailIds = mails.Select(m => m.Id).ToHashSet();
|
||||
var threadMails = await GetMailsByThreadIdsAsync(uniqueThreadIds, existingMailIds).ConfigureAwait(false);
|
||||
|
||||
if (threadMails?.Count > 0)
|
||||
{
|
||||
// Get all thread mails in a single DB call
|
||||
var existingMailIds = expandedMails.Select(m => m.Id).ToHashSet();
|
||||
var allThreadMails = await GetMailsByThreadIdsAsync(uniqueThreadIds, existingMailIds).ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (allThreadMails?.Count > 0)
|
||||
// Load any folders that thread mails belong to but are not yet cached.
|
||||
var newFolderIds = threadMails
|
||||
.Select(m => m.FolderId)
|
||||
.Distinct()
|
||||
.Where(id => !folderCache.ContainsKey(id))
|
||||
.ToList();
|
||||
|
||||
if (newFolderIds.Count > 0)
|
||||
{
|
||||
// Process thread mails in parallel to improve performance
|
||||
var tasks = allThreadMails.Select(async threadMail =>
|
||||
{
|
||||
await LoadAssignedPropertiesWithCacheAsync(threadMail, folderCache, accountCache, contactCache).ConfigureAwait(false);
|
||||
return threadMail;
|
||||
});
|
||||
var newFolders = await Task.WhenAll(
|
||||
newFolderIds.Select(id => _folderService.GetFolderAsync(id))).ConfigureAwait(false);
|
||||
|
||||
var processedThreadMails = await Task.WhenAll(tasks).ConfigureAwait(false);
|
||||
|
||||
// Filter out items with no assigned account or folder
|
||||
var validThreadMails = processedThreadMails.Where(m => m.AssignedAccount != null && m.AssignedFolder != null);
|
||||
|
||||
expandedMails.AddRange(validThreadMails);
|
||||
foreach (var f in newFolders.Where(f => f != null))
|
||||
folderCache[f.Id] = f;
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// Batch-fetch contacts for any new senders in thread mails.
|
||||
var newAddresses = threadMails
|
||||
.Where(m => !string.IsNullOrEmpty(m.FromAddress) && !contactCache.ContainsKey(m.FromAddress))
|
||||
.Select(m => m.FromAddress)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
if (newAddresses.Count > 0)
|
||||
{
|
||||
var newContacts = await _contactService.GetContactsByAddressesAsync(newAddresses).ConfigureAwait(false);
|
||||
foreach (var c in newContacts.Where(c => c != null))
|
||||
contactCache[c.Address] = c;
|
||||
}
|
||||
|
||||
AssignPropertiesFromCaches(threadMails, folderCache, accountCache, contactCache);
|
||||
mails.AddRange(threadMails.Where(m => m.AssignedAccount != null && m.AssignedFolder != null));
|
||||
}
|
||||
|
||||
return [.. expandedMails];
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return [.. mails];
|
||||
}
|
||||
|
||||
private async Task<List<MailCopy>> GetMailsByThreadIdAsync(string threadId, HashSet<string> excludeMailIds)
|
||||
/// <summary>
|
||||
/// Assigns AssignedFolder, AssignedAccount, and SenderContact to each mail from pre-loaded
|
||||
/// in-memory dictionaries. No DB calls are made here.
|
||||
/// </summary>
|
||||
private void AssignPropertiesFromCaches(
|
||||
List<MailCopy> mails,
|
||||
Dictionary<Guid, MailItemFolder> folderCache,
|
||||
Dictionary<Guid, MailAccount> accountCache,
|
||||
Dictionary<string, AccountContact> contactCache)
|
||||
{
|
||||
if (string.IsNullOrEmpty(threadId))
|
||||
return [];
|
||||
foreach (var mail in mails)
|
||||
{
|
||||
if (!folderCache.TryGetValue(mail.FolderId, out var folder))
|
||||
continue;
|
||||
|
||||
var placeholders = string.Join(",", excludeMailIds.Select(_ => "?"));
|
||||
var sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId = ? AND Id NOT IN ({placeholders})";
|
||||
var parameters = new List<object> { threadId };
|
||||
parameters.AddRange(excludeMailIds.Cast<object>());
|
||||
if (!accountCache.TryGetValue(folder.MailAccountId, out var account))
|
||||
continue;
|
||||
|
||||
return await Connection.QueryAsync<MailCopy>(sql, parameters.ToArray());
|
||||
mail.AssignedFolder = folder;
|
||||
mail.AssignedAccount = account;
|
||||
|
||||
// Self-sent mails (e.g. Sent folder): construct contact from account meta
|
||||
// to get the up-to-date profile picture without a DB roundtrip.
|
||||
if (!string.IsNullOrEmpty(mail.FromAddress) && mail.FromAddress == account.Address)
|
||||
{
|
||||
mail.SenderContact = new AccountContact
|
||||
{
|
||||
Address = account.Address,
|
||||
Name = account.SenderName,
|
||||
Base64ContactPicture = account.Base64ProfilePictureData
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
contactCache.TryGetValue(mail.FromAddress ?? string.Empty, out var contact);
|
||||
mail.SenderContact = contact ?? CreateUnknownContact(mail.FromName, mail.FromAddress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<MailCopy>> GetMailsByThreadIdsAsync(List<string> threadIds, HashSet<string> excludeMailIds)
|
||||
@@ -417,63 +488,24 @@ public class MailService : BaseDatabaseService, IMailService
|
||||
return [];
|
||||
|
||||
var threadPlaceholders = string.Join(",", threadIds.Select(_ => "?"));
|
||||
var excludePlaceholders = string.Join(",", excludeMailIds.Select(_ => "?"));
|
||||
var sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders}) AND Id NOT IN ({excludePlaceholders})";
|
||||
var parameters = new List<object>();
|
||||
parameters.AddRange(threadIds.Cast<object>());
|
||||
parameters.AddRange(excludeMailIds.Cast<object>());
|
||||
|
||||
string sql;
|
||||
if (excludeMailIds.Count > 0)
|
||||
{
|
||||
var excludePlaceholders = string.Join(",", excludeMailIds.Select(_ => "?"));
|
||||
sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders}) AND Id NOT IN ({excludePlaceholders})";
|
||||
parameters.AddRange(excludeMailIds.Cast<object>());
|
||||
}
|
||||
else
|
||||
{
|
||||
sql = $"SELECT MailCopy.* FROM MailCopy WHERE ThreadId IN ({threadPlaceholders})";
|
||||
}
|
||||
|
||||
return await Connection.QueryAsync<MailCopy>(sql, parameters.ToArray()).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This method should used for operations with multiple mailItems. Don't use this for single mail items.
|
||||
/// Called method should provide own instances for caches.
|
||||
/// </summary>
|
||||
private async Task LoadAssignedPropertiesWithCacheAsync(MailCopy mail, ConcurrentDictionary<Guid, MailItemFolder> folderCache, ConcurrentDictionary<Guid, MailAccount> accountCache, ConcurrentDictionary<string, AccountContact> contactCache)
|
||||
{
|
||||
if (mail is MailCopy mailCopy)
|
||||
{
|
||||
var isFolderCached = folderCache.TryGetValue(mailCopy.FolderId, out MailItemFolder folderAssignment);
|
||||
MailAccount accountAssignment = null;
|
||||
if (!isFolderCached)
|
||||
{
|
||||
folderAssignment = await _folderService.GetFolderAsync(mailCopy.FolderId).ConfigureAwait(false);
|
||||
folderCache.TryAdd(mailCopy.FolderId, folderAssignment);
|
||||
}
|
||||
|
||||
if (folderAssignment != null)
|
||||
{
|
||||
var isAccountCached = accountCache.TryGetValue(folderAssignment.MailAccountId, out accountAssignment);
|
||||
if (!isAccountCached)
|
||||
{
|
||||
accountAssignment = await _accountService.GetAccountAsync(folderAssignment.MailAccountId).ConfigureAwait(false);
|
||||
|
||||
accountCache.TryAdd(folderAssignment.MailAccountId, accountAssignment);
|
||||
}
|
||||
}
|
||||
|
||||
AccountContact contactAssignment = null;
|
||||
|
||||
bool isContactCached = !string.IsNullOrEmpty(mailCopy.FromAddress) &&
|
||||
contactCache.TryGetValue(mailCopy.FromAddress, out contactAssignment);
|
||||
|
||||
if (!isContactCached && accountAssignment != null)
|
||||
{
|
||||
contactAssignment = await GetSenderContactForAccountAsync(accountAssignment, mailCopy.FromAddress).ConfigureAwait(false);
|
||||
|
||||
if (contactAssignment != null)
|
||||
{
|
||||
contactCache.TryAdd(mailCopy.FromAddress, contactAssignment);
|
||||
}
|
||||
}
|
||||
|
||||
mailCopy.AssignedFolder = folderAssignment;
|
||||
mailCopy.AssignedAccount = accountAssignment;
|
||||
mailCopy.SenderContact = contactAssignment ?? CreateUnknownContact(mailCopy.FromName, mailCopy.FromAddress);
|
||||
}
|
||||
}
|
||||
|
||||
private static AccountContact CreateUnknownContact(string fromName, string fromAddress)
|
||||
{
|
||||
if (string.IsNullOrEmpty(fromName) && string.IsNullOrEmpty(fromAddress))
|
||||
@@ -1407,14 +1439,19 @@ public class MailService : BaseDatabaseService, IMailService
|
||||
var mailCopies = await Connection.QueryAsync<MailCopy>(sql, mailCopyIds.Cast<object>().ToArray());
|
||||
if (mailCopies?.Count == 0) return [];
|
||||
|
||||
ConcurrentDictionary<Guid, MailItemFolder> folderCache = new();
|
||||
ConcurrentDictionary<Guid, MailAccount> accountCache = new();
|
||||
ConcurrentDictionary<string, AccountContact> contactCache = new();
|
||||
var folderIds = mailCopies.Select(m => m.FolderId).Distinct().ToList();
|
||||
var folderTasks = folderIds.Select(id => _folderService.GetFolderAsync(id));
|
||||
var folders = await Task.WhenAll(folderTasks).ConfigureAwait(false);
|
||||
var folderCache = folders.Where(f => f != null).ToDictionary(f => f.Id);
|
||||
|
||||
foreach (var mail in mailCopies)
|
||||
{
|
||||
await LoadAssignedPropertiesWithCacheAsync(mail, folderCache, accountCache, contactCache).ConfigureAwait(false);
|
||||
}
|
||||
var allAccounts = await _accountService.GetAccountsAsync().ConfigureAwait(false);
|
||||
var accountCache = allAccounts.ToDictionary(a => a.Id);
|
||||
|
||||
var addresses = mailCopies.Where(m => !string.IsNullOrEmpty(m.FromAddress)).Select(m => m.FromAddress).Distinct().ToList();
|
||||
var contactList = await _contactService.GetContactsByAddressesAsync(addresses).ConfigureAwait(false);
|
||||
var contactCache = contactList.ToDictionary(c => c.Address);
|
||||
|
||||
AssignPropertiesFromCaches(mailCopies, folderCache, accountCache, contactCache);
|
||||
|
||||
return mailCopies;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user