javier/Wino-Mail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add initial mail sync range selection

Browse Source
This commit is contained in:
Burak Kaan Köse
2026-04-14 00:03:48 +02:00
parent 2e36772a4c
commit c622858d2d
21 changed files with 378 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Wino.Core/Synchronizers/GmailSynchronizer.cs
+11 -14
View File
@@ -81,9 +81,8 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
public override uint BatchModificationSize => 1000;
/// <summary>
/// Maximum messages to fetch per folder during initial sync (1500).
/// All messages are downloaded with METADATA ONLY - no raw MIME content.
/// Uses Gmail API's Metadata format which includes headers, labels, and snippet but NOT full message body.
/// Legacy page size hint kept for compatibility with shared synchronizer contracts.
/// Gmail initial sync now downloads all messages inside the selected cutoff window.
/// </summary>
public override uint InitialMessageDownloadCountPerFolder => 1500;
@@ -304,13 +303,18 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
/// <summary>
/// Performs initial synchronization by downloading messages per-folder.
/// Each folder gets up to 1500 messages, but we track already downloaded message IDs globally
/// to avoid downloading the same message multiple times (Gmail messages can have multiple labels).
/// Messages are filtered by the account's configured initial synchronization cutoff date when present,
/// and duplicates are avoided globally because Gmail messages can have multiple labels.
/// </summary>
private async Task<List<string>> PerformInitialSyncAsync(CancellationToken cancellationToken)
{
// Track all downloaded message IDs globally to avoid duplicate downloads
var downloadedMessageIds = new HashSet<string>();
var referenceDateUtc = Account.CreatedAt ?? DateTime.UtcNow;
var initialSynchronizationCutoffDateUtc = Account.InitialSynchronizationRange.ToCutoffDateUtc(referenceDateUtc);
var queryText = initialSynchronizationCutoffDateUtc.HasValue
? $"after:{initialSynchronizationCutoffDateUtc.Value.ToUniversalTime():yyyy/MM/dd}"
: null;
_logger.Information("Performing initial sync for {Name} - downloading messages per folder", Account.Name);
@@ -337,7 +341,6 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
var folderDownloaded = 0;
string pageToken = null;
var remainingToDownload = (int)InitialMessageDownloadCountPerFolder;
do
{
@@ -345,8 +348,9 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
var request = _gmailService.Users.Messages.List("me");
request.LabelIds = new Google.Apis.Util.Repeatable<string>(new[] { folder.RemoteFolderId });
request.MaxResults = Math.Min(remainingToDownload, 500); // API max is 500
request.MaxResults = 500; // API max is 500
request.PageToken = pageToken;
request.Q = queryText;
var response = await request.ExecuteAsync(cancellationToken);
@@ -373,19 +377,12 @@ public class GmailSynchronizer : WinoSynchronizer<IClientServiceRequest, Message
totalMessagesDownloaded += newMessageIds.Count;
}
// Count all messages (including duplicates) toward the folder limit
remainingToDownload -= response.Messages.Count;
_logger.Debug("Folder {FolderName}: Downloaded {New} new messages ({Total} total in folder)",
folder.FolderName, newMessageIds.Count, folderDownloaded);
}
pageToken = response.NextPageToken;
// Stop if we've processed enough messages for this folder or no more pages
if (remainingToDownload <= 0 || string.IsNullOrEmpty(pageToken))
break;
} while (!string.IsNullOrEmpty(pageToken));
_logger.Information("Folder {FolderName}: Downloaded {Count} messages", folder.FolderName, folderDownloaded);
Wino.Core/Synchronizers/ImapSync/UnifiedImapSynchronizer.cs
+49 -23
View File
@@ -9,6 +9,7 @@ using MailKit.Search;
using MoreLinq;
using Serilog;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Extensions;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.MailItem;
@@ -252,9 +253,20 @@ public class UnifiedImapSynchronizer
.OpenAsync(FolderAccess.ReadOnly, folder.UidValidity, localHighestModSeq, knownUidStructs, cancellationToken)
.ConfigureAwait(false);
var changedUids = await remoteFolder
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
.ConfigureAwait(false);
IList<UniqueId> changedUids;
if (folder.HighestModeSeq == 0)
{
changedUids = await remoteFolder
.SearchAsync(BuildInitialSyncQuery(synchronizer), cancellationToken)
.ConfigureAwait(false);
}
else
{
changedUids = await remoteFolder
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
.ConfigureAwait(false);
}
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, changedUids, synchronizer, cancellationToken).ConfigureAwait(false);
@@ -308,25 +320,26 @@ public class UnifiedImapSynchronizer
{
IList<UniqueId> changedUids;
if (client.Capabilities.HasFlag(ImapCapabilities.Sort))
if (isInitialSync)
{
changedUids = await remoteFolder
.SortAsync(SearchQuery.ChangedSince(localHighestModSeq), [OrderBy.ReverseDate], cancellationToken)
.SearchAsync(BuildInitialSyncQuery(synchronizer), cancellationToken)
.ConfigureAwait(false);
}
else
{
changedUids = await remoteFolder
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
.ConfigureAwait(false);
}
if (isInitialSync)
{
changedUids = changedUids
.OrderByDescending(a => a.Id)
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
.ToList();
if (client.Capabilities.HasFlag(ImapCapabilities.Sort))
{
changedUids = await remoteFolder
.SortAsync(SearchQuery.ChangedSince(localHighestModSeq), [OrderBy.ReverseDate], cancellationToken)
.ConfigureAwait(false);
}
else
{
changedUids = await remoteFolder
.SearchAsync(SearchQuery.ChangedSince(localHighestModSeq), cancellationToken)
.ConfigureAwait(false);
}
}
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, changedUids, synchronizer, cancellationToken).ConfigureAwait(false);
@@ -367,15 +380,12 @@ public class UnifiedImapSynchronizer
if (folder.HighestKnownUid == 0)
{
var remoteUids = await remoteFolder.SearchAsync(SearchQuery.All, cancellationToken).ConfigureAwait(false);
var initialUids = remoteUids
.OrderByDescending(a => a.Id)
.Take((int)synchronizer.InitialMessageDownloadCountPerFolder)
.ToList();
var initialUids = await remoteFolder
.SearchAsync(BuildInitialSyncQuery(synchronizer), cancellationToken)
.ConfigureAwait(false);
downloadedMessageIds = await DownloadMessagesByUidsAsync(client, remoteFolder, folder, initialUids, synchronizer, cancellationToken).ConfigureAwait(false);
UpdateHighestKnownUid(folder, remoteFolder, remoteUids.Select(a => a.Id));
UpdateHighestKnownUid(folder, remoteFolder, initialUids.Select(a => a.Id));
}
else
{
@@ -410,6 +420,22 @@ public class UnifiedImapSynchronizer
#region Shared Helpers
private static SearchQuery BuildInitialSyncQuery(IImapSynchronizer synchronizer)
{
if (synchronizer is IBaseSynchronizer { Account: { } account })
{
var referenceDateUtc = account.CreatedAt ?? DateTime.UtcNow;
var cutoffDateUtc = account.InitialSynchronizationRange.ToCutoffDateUtc(referenceDateUtc);
if (cutoffDateUtc.HasValue)
{
return SearchQuery.DeliveredAfter(cutoffDateUtc.Value.ToUniversalTime().Date);
}
}
return SearchQuery.All;
}
private async Task EnsureUidValidityStateAsync(MailItemFolder folder, IMailFolder remoteFolder)
{
if (folder.UidValidity != 0 && remoteFolder.UidValidity != folder.UidValidity)
Wino.Core/Synchronizers/OutlookSynchronizer.cs
+23 -13
View File
@@ -55,14 +55,14 @@ public partial class OutlookSynchronizerJsonContext : JsonSerializerContext;
///
/// SYNCHRONIZATION STRATEGY:
/// - Uses delta API for both initial and incremental sync
/// - Initial sync: Downloads last 30 days of emails with metadata only
/// - Initial sync: Downloads messages using the account's configured cutoff date with metadata only
/// - Incremental sync: Uses delta token to get only changes since last sync
/// - Messages are downloaded with metadata only (no MIME content during sync)
/// - MIME files are downloaded on-demand when user explicitly reads a message
///
/// Key implementation details:
/// - SynchronizeFolderAsync: Main entry point for per-folder synchronization
/// - DownloadMailsForInitialSyncAsync: Downloads last 30 days using delta API with filter
/// - DownloadMailsForInitialSyncAsync: Downloads messages using delta API with an optional cutoff filter
/// - ProcessDeltaChangesAsync: Processes incremental changes using delta token
/// - DownloadMessageMetadataBatchAsync: Downloads metadata in batches using Graph batch API
/// - CreateMailCopyFromMessageAsync: Creates MailCopy from Message metadata
@@ -343,9 +343,9 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
// Check if we have a delta token
if (string.IsNullOrEmpty(folder.DeltaToken))
{
_logger.Debug("No delta token for folder {FolderName}. Starting initial sync (last 30 days).", folder.FolderName);
_logger.Debug("No delta token for folder {FolderName}. Starting initial sync.", folder.FolderName);
// Download mails for initial sync (last 30 days)
// Download mails for initial sync using the account's configured cutoff date.
await DownloadMailsForInitialSyncAsync(folder, downloadedMessageIds, cancellationToken).ConfigureAwait(false);
}
else
@@ -367,27 +367,37 @@ public class OutlookSynchronizer : WinoSynchronizer<RequestInformation, Message,
}
/// <summary>
/// Downloads mails for initial synchronization using Delta API with 30-day filter.
/// Downloads metadata only (no MIME content) for messages received in the last 30 days.
/// Downloads mails for initial synchronization using Delta API with the account's configured cutoff date.
/// Downloads metadata only (no MIME content) for messages received after that date.
/// </summary>
private async Task DownloadMailsForInitialSyncAsync(MailItemFolder folder, List<string> downloadedMessageIds, CancellationToken cancellationToken)
{
_logger.Debug("Starting initial mail download for folder {FolderName} (last 6 months)", folder.FolderName);
_logger.Debug("Starting initial mail download for folder {FolderName}", folder.FolderName);
try
{
// Calculate date 6 months ago
var sixMonthsAgo = DateTime.UtcNow.AddMonths(-6);
var filterDate = sixMonthsAgo.ToString("yyyy-MM-ddTHH:mm:ssZ");
var referenceDateUtc = Account.CreatedAt ?? DateTime.UtcNow;
var initialSynchronizationCutoffDateUtc = Account.InitialSynchronizationRange.ToCutoffDateUtc(referenceDateUtc);
var filterDate = initialSynchronizationCutoffDateUtc?.ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ");
_logger.Information("Downloading messages received after {FilterDate} for folder {FolderName}", filterDate, folder.FolderName);
if (filterDate != null)
{
_logger.Information("Downloading messages received after {FilterDate} for folder {FolderName}", filterDate, folder.FolderName);
}
else
{
_logger.Information("Downloading all available messages for folder {FolderName}", folder.FolderName);
}
// Use Delta API with receivedDateTime filter for last 6 months
var messageCollectionPage = await _graphClient.Me.MailFolders[folder.RemoteFolderId].Messages.Delta.GetAsDeltaGetResponseAsync((config) =>
{
config.QueryParameters.Select = outlookMessageSelectParameters;
config.QueryParameters.Orderby = ["receivedDateTime desc"];
config.QueryParameters.Filter = $"receivedDateTime ge {filterDate}";
if (filterDate != null)
{
config.QueryParameters.Filter = $"receivedDateTime ge {filterDate}";
}
}, cancellationToken).ConfigureAwait(false);
var totalProcessed = 0;