using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using CommunityToolkit.Mvvm.Messaging;
using Google;
using Google.Apis.Calendar.v3.Data;
using Google.Apis.Gmail.v1;
using Google.Apis.Gmail.v1.Data;
using Google.Apis.Http;
using Google.Apis.PeopleService.v1;
using Google.Apis.Requests;
using Google.Apis.Services;
using Google.Apis.Upload;
using MailKit;
using Microsoft.IdentityModel.Tokens;
using MimeKit;
using MoreLinq;
using Serilog;
using Wino.Core.Domain;
using Wino.Core.Domain.Entities.Calendar;
using Wino.Core.Domain.Entities.Mail;
using Wino.Core.Domain.Entities.Shared;
using Wino.Core.Domain.Enums;
using Wino.Core.Domain.Exceptions;
using Wino.Core.Domain.Extensions;
using Wino.Core.Domain.Interfaces;
using Wino.Core.Domain.Models.Accounts;
using Wino.Core.Domain.Models.Folders;
using Wino.Core.Domain.Models.MailItem;
using Wino.Core.Domain.Models.Synchronization;
using Wino.Core.Extensions;
using Wino.Core.Http;
using Wino.Core.Integration.Processors;
using Wino.Core.Misc;
using Wino.Core.Requests.Bundles;
using Wino.Core.Requests.Calendar;
using Wino.Core.Requests.Folder;
using Wino.Core.Requests.Mail;
using Wino.Messaging.UI;
using Wino.Services;
using CalendarService = Google.Apis.Calendar.v3.CalendarService;
using DriveFile = Google.Apis.Drive.v3.Data.File;
using DriveService = Google.Apis.Drive.v3.DriveService;
namespace Wino.Core.Synchronizers.Mail;
[JsonSerializable(typeof(Message))]
[JsonSerializable(typeof(Label))]
[JsonSerializable(typeof(Draft))]
[JsonSerializable(typeof(Event))]
public partial class GmailSynchronizerJsonContext : JsonSerializerContext;
///
/// Gmail synchronizer implementation using Gmail History API for efficient incremental sync.
///
/// SYNCHRONIZATION STRATEGY:
/// - Initial sync: Downloads up to 15c00 messages PER FOLDER with metadata only.
/// Uses a global HashSet to track downloaded message IDs, avoiding duplicate downloads
/// when messages have multiple labels. Each folder gets its full quota of messages.
/// - Incremental sync: Uses ONLY History API to get changes since last sync.
/// No per-folder downloads during incremental sync - this is the proper Gmail sync approach.
/// - Messages are downloaded with metadata only during initial sync (no MIME content)
/// - New messages during incremental sync are downloaded with full MIME content
/// - MIME files for initial sync messages are downloaded on-demand when user reads a message
///
/// Key implementation details:
/// - PerformInitialSyncAsync: Downloads messages per-folder with global deduplication
/// - SynchronizeDeltaAsync: Processes incremental changes using History API with pagination
/// - Handles 404/410 errors (history expired) by triggering full resync
/// - CreateMinimalMailCopyAsync: Extracts MailCopy fields from Gmail Metadata format
/// - DownloadMissingMimeMessageAsync: Downloads raw MIME only when explicitly requested
///
public class GmailSynchronizer : WinoSynchronizer, IHttpClientFactory
{
public override uint BatchModificationSize => 1000;
///
/// Legacy page size hint kept for compatibility with shared synchronizer contracts.
/// Gmail initial sync now downloads all messages inside the selected cutoff window.
///
public override uint InitialMessageDownloadCountPerFolder => 1500;
// It's actually 100. But Gmail SDK has internal bug for Out of Memory exception.
// https://github.com/googleapis/google-api-dotnet-client/issues/2603
private const uint MaximumAllowedBatchRequestSize = 10;
private readonly ConfigurableHttpClient _googleHttpClient;
private readonly GmailService _gmailService;
private readonly CalendarService _calendarService;
private readonly DriveService _driveService;
private readonly PeopleServiceService _peopleService;
private readonly IGmailChangeProcessor _gmailChangeProcessor;
private readonly IGmailSynchronizerErrorHandlerFactory _gmailSynchronizerErrorHandlerFactory;
private readonly ILogger _logger = Log.ForContext();
// Keeping a reference for quick access to the virtual archive folder.
private Guid? archiveFolderId;
private bool _isFolderStructureChanged;
public GmailSynchronizer(MailAccount account,
IGmailAuthenticator authenticator,
IGmailChangeProcessor gmailChangeProcessor,
IGmailSynchronizerErrorHandlerFactory gmailSynchronizerErrorHandlerFactory) : base(account, WeakReferenceMessenger.Default)
{
var messageHandler = new GmailClientMessageHandler(authenticator, account);
var initializer = new BaseClientService.Initializer()
{
HttpClientFactory = this
};
_googleHttpClient = new ConfigurableHttpClient(messageHandler);
_gmailService = new GmailService(initializer);
_peopleService = new PeopleServiceService(initializer);
_calendarService = new CalendarService(initializer);
_driveService = new DriveService(initializer);
_gmailChangeProcessor = gmailChangeProcessor;
_gmailSynchronizerErrorHandlerFactory = gmailSynchronizerErrorHandlerFactory;
}
public ConfigurableHttpClient CreateHttpClient(CreateHttpClientArgs args) => _googleHttpClient;
public override async Task GetProfileInformationAsync()
{
var profileRequest = _peopleService.People.Get("people/me");
profileRequest.PersonFields = "names,photos,emailAddresses";
string senderName = string.Empty, base64ProfilePicture = string.Empty, address = string.Empty;
var userProfile = await profileRequest.ExecuteAsync();
senderName = userProfile.Names?.FirstOrDefault()?.DisplayName ?? Account.SenderName;
var profilePicture = userProfile.Photos?.FirstOrDefault()?.Url ?? string.Empty;
if (!string.IsNullOrEmpty(profilePicture))
{
base64ProfilePicture = await GetProfilePictureBase64EncodedAsync(profilePicture).ConfigureAwait(false);
}
address = userProfile.EmailAddresses.FirstOrDefault(a => a.Metadata.Primary == true).Value;
return new ProfileInformation(senderName, base64ProfilePicture, address);
}
protected override async Task SynchronizeAliasesAsync()
{
var sendAsListRequest = _gmailService.Users.Settings.SendAs.List("me");
var sendAsListResponse = await sendAsListRequest.ExecuteAsync();
var remoteAliases = sendAsListResponse.GetRemoteAliases();
await _gmailChangeProcessor.UpdateRemoteAliasInformationAsync(Account, remoteAliases).ConfigureAwait(false);
}
protected override async Task SynchronizeMailsInternalAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default)
{
_logger.Information("Internal mail synchronization started for {Name}", Account.Name);
var downloadedMessageIds = new List();
var folderResults = new List();
try
{
_isFolderStructureChanged = false;
// Make sure that virtual archive folder exists before all.
if (!archiveFolderId.HasValue)
await InitializeArchiveFolderAsync().ConfigureAwait(false);
// Gmail must always synchronize folders before because it doesn't have a per-folder sync.
_logger.Information("Synchronizing folders for {Name}", Account.Name);
UpdateSyncProgress(0, 0, "Synchronizing folders...");
try
{
await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false);
}
catch (GoogleApiException googleException) when (googleException.Message.Contains("Mail service not enabled"))
{
throw new GmailServiceDisabledException();
}
if (_isFolderStructureChanged)
{
WeakReferenceMessenger.Default.Send(new AccountFolderConfigurationUpdated(Account.Id));
}
_logger.Information("Synchronizing folders for {Name} is completed", Account.Name);
UpdateSyncProgress(0, 0, "Folders synchronized");
// Stop synchronization at this point if type is only folder metadata sync.
if (options.Type == MailSynchronizationType.FoldersOnly) return MailSynchronizationResult.Empty;
cancellationToken.ThrowIfCancellationRequested();
bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier);
_logger.Debug("Is initial synchronization: {IsInitialSync}", isInitialSync);
if (isInitialSync)
{
// INITIAL SYNC: Download all messages globally (not per-folder) to avoid duplicates.
// Gmail messages can have multiple labels, so per-folder download would fetch same message multiple times.
downloadedMessageIds = await PerformInitialSyncAsync(cancellationToken).ConfigureAwait(false);
// Set the history ID to the latest value after initial sync
UpdateSyncProgress(0, 0, "Finalizing synchronization...");
var profile = await _gmailService.Users.GetProfile("me").ExecuteAsync(cancellationToken);
if (profile.HistoryId.HasValue)
{
await UpdateAccountSyncIdentifierAsync(profile.HistoryId.Value).ConfigureAwait(false);
_logger.Information("Initial sync completed. Set history ID to {HistoryId}", profile.HistoryId.Value);
}
// Create successful folder results for all folders
var allFolders = await _gmailChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false);
foreach (var folder in allFolders.Where(f => f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID))
{
folderResults.Add(FolderSyncResult.Successful(folder.Id, folder.FolderName, 0));
}
}
else
{
// INCREMENTAL SYNC: Use ONLY History API - no per-folder downloads.
// This is the proper Gmail sync strategy as recommended by Google.
UpdateSyncProgress(0, 0, "Synchronizing changes...");
var deltaResult = await SynchronizeDeltaAsync(options, cancellationToken).ConfigureAwait(false);
downloadedMessageIds.AddRange(deltaResult.DownloadedMessageIds);
// If history sync was reset due to expired history ID, we need to do initial sync
if (deltaResult.RequiresFullResync)
{
_logger.Warning("History ID expired. Performing full resync for {Name}", Account.Name);
downloadedMessageIds = await PerformInitialSyncAsync(cancellationToken).ConfigureAwait(false);
// Update history ID after full resync
var profile = await _gmailService.Users.GetProfile("me").ExecuteAsync(cancellationToken);
if (profile.HistoryId.HasValue)
{
await UpdateAccountSyncIdentifierAsync(profile.HistoryId.Value).ConfigureAwait(false);
_logger.Information("Full resync completed. Set history ID to {HistoryId}", profile.HistoryId.Value);
}
}
UpdateSyncProgress(0, 0, "Changes synchronized");
// Create folder results for incremental sync
var allFolders = await _gmailChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false);
foreach (var folder in allFolders.Where(f => f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID))
{
folderResults.Add(FolderSyncResult.Successful(folder.Id, folder.FolderName, 0));
}
}
// Map Gmail Draft resource IDs for all drafts.
// Gmail's Messages API doesn't expose Draft IDs, so we query the Drafts API separately.
// This ensures DraftId is correctly set for both Wino-created and externally-created drafts.
await MapDraftIdsAsync(cancellationToken).ConfigureAwait(false);
// Keep virtual Archive folder assignments in sync with Gmail "in:archive" query.
try
{
await MapArchivedMailsAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.Warning(ex, "Failed to map Gmail archive folder for {Name}", Account.Name);
}
}
catch (OperationCanceledException)
{
_logger.Information("Synchronization was canceled for {Name}", Account.Name);
return MailSynchronizationResult.Canceled;
}
catch (Exception ex)
{
_logger.Error(ex, "Synchronization failed for {Name}", Account.Name);
return MailSynchronizationResult.Failed(ex);
}
// Get all unread new downloaded items for notifications
var unreadNewItems = await _gmailChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false);
return MailSynchronizationResult.CompletedWithFolderResults(unreadNewItems, folderResults);
}
///
/// Result of delta synchronization using History API.
///
private record DeltaSyncResult(List DownloadedMessageIds, bool RequiresFullResync);
///
/// Performs initial synchronization by downloading messages per-folder.
/// Messages are filtered by the account's configured initial synchronization cutoff date when present,
/// and duplicates are avoided globally because Gmail messages can have multiple labels.
///
private async Task> PerformInitialSyncAsync(CancellationToken cancellationToken)
{
// Track all downloaded message IDs globally to avoid duplicate downloads
var downloadedMessageIds = new HashSet();
var referenceDateUtc = Account.CreatedAt ?? DateTime.UtcNow;
var initialSynchronizationCutoffDateUtc = Account.InitialSynchronizationRange.ToCutoffDateUtc(referenceDateUtc);
var queryText = initialSynchronizationCutoffDateUtc.HasValue
? $"after:{initialSynchronizationCutoffDateUtc.Value.ToUniversalTime():yyyy/MM/dd}"
: null;
_logger.Information("Performing initial sync for {Name} - downloading messages per folder", Account.Name);
try
{
// Get all folders to sync (exclude virtual ARCHIVE folder)
var folders = await _gmailChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
var syncableFolders = folders
.Where(f => f.IsSynchronizationEnabled && f.RemoteFolderId != ServiceConstants.ARCHIVE_LABEL_ID)
.OrderByDescending(f => f.SpecialFolderType == SpecialFolderType.Draft || f.RemoteFolderId == ServiceConstants.DRAFT_LABEL_ID)
.ToList();
var totalFolders = syncableFolders.Count;
var totalMessagesDownloaded = 0;
for (int i = 0; i < totalFolders; i++)
{
var folder = syncableFolders[i];
cancellationToken.ThrowIfCancellationRequested();
UpdateSyncProgress(totalFolders, totalFolders - (i + 1), $"Syncing {folder.FolderName}...");
_logger.Debug("Downloading messages for folder {FolderName} (label: {LabelId})", folder.FolderName, folder.RemoteFolderId);
var folderDownloaded = 0;
string pageToken = null;
do
{
cancellationToken.ThrowIfCancellationRequested();
var request = _gmailService.Users.Messages.List("me");
request.LabelIds = new Google.Apis.Util.Repeatable(new[] { folder.RemoteFolderId });
request.MaxResults = 500; // API max is 500
request.PageToken = pageToken;
request.Q = queryText;
var response = await request.ExecuteAsync(cancellationToken);
if (response.Messages != null && response.Messages.Count > 0)
{
// Filter out already downloaded messages to avoid duplicates
var newMessageIds = response.Messages
.Select(m => m.Id)
.Where(id => !downloadedMessageIds.Contains(id))
.ToList();
if (newMessageIds.Count > 0)
{
// Draft folder needs MIME during initial sync so compose can open immediately.
bool shouldDownloadRawMime = folder.SpecialFolderType == SpecialFolderType.Draft || folder.RemoteFolderId == ServiceConstants.DRAFT_LABEL_ID;
await DownloadMessagesInBatchAsync(newMessageIds, downloadRawMime: shouldDownloadRawMime, cancellationToken).ConfigureAwait(false);
foreach (var id in newMessageIds)
{
downloadedMessageIds.Add(id);
}
folderDownloaded += newMessageIds.Count;
totalMessagesDownloaded += newMessageIds.Count;
}
_logger.Debug("Folder {FolderName}: Downloaded {New} new messages ({Total} total in folder)",
folder.FolderName, newMessageIds.Count, folderDownloaded);
}
pageToken = response.NextPageToken;
} while (!string.IsNullOrEmpty(pageToken));
_logger.Information("Folder {FolderName}: Downloaded {Count} messages", folder.FolderName, folderDownloaded);
UpdateSyncProgress(totalFolders, 0, Translator.SyncAction_SynchronizingAccount);
}
_logger.Information("Initial sync completed. Downloaded {Count} unique messages for {Name}", downloadedMessageIds.Count, Account.Name);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
_logger.Warning("Rate limit exceeded during initial sync. Retrying after delay.");
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
throw;
}
catch (Exception ex)
{
_logger.Error(ex, "Error during initial sync for {Name}", Account.Name);
throw;
}
return downloadedMessageIds.ToList();
}
///
/// Performs incremental synchronization using Gmail History API.
/// This is the recommended approach for Gmail sync after initial sync is complete.
/// Returns a result indicating downloaded messages and whether a full resync is needed.
///
private async Task SynchronizeDeltaAsync(MailSynchronizationOptions options, CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List();
try
{
string pageToken = null;
do
{
cancellationToken.ThrowIfCancellationRequested();
var historyRequest = _gmailService.Users.History.List("me");
historyRequest.StartHistoryId = ulong.Parse(Account.SynchronizationDeltaIdentifier!);
if (!string.IsNullOrEmpty(pageToken))
historyRequest.PageToken = pageToken;
var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
if (historyResponse.History != null)
{
var addedMessageIds = new List();
// Collect all added messages first
foreach (var historyRecord in historyResponse.History)
{
if (historyRecord.MessagesAdded != null)
{
addedMessageIds.AddRange(historyRecord.MessagesAdded.Select(ma => ma.Message.Id));
}
}
// Process added messages in batches if any
// During delta sync, download with Raw format to get MIME content for new messages
if (addedMessageIds.Count != 0)
{
// Deduplicate message IDs
var uniqueAddedIds = addedMessageIds.Distinct().ToList();
await DownloadMessagesInBatchAsync(uniqueAddedIds, downloadRawMime: true, cancellationToken).ConfigureAwait(false);
downloadedMessageIds.AddRange(uniqueAddedIds);
}
// Process other history changes (label changes, deletions)
await ProcessHistoryChangesAsync(historyResponse).ConfigureAwait(false);
}
// CRITICAL: Update the history ID to the latest one after processing all changes
// History IDs are always incremental, so the response contains the latest history ID
if (historyResponse.HistoryId.HasValue)
{
await UpdateAccountSyncIdentifierAsync(historyResponse.HistoryId.Value).ConfigureAwait(false);
_logger.Debug("Updated history ID to {HistoryId} after delta sync", historyResponse.HistoryId.Value);
}
pageToken = historyResponse.NextPageToken;
} while (!string.IsNullOrEmpty(pageToken));
_logger.Information("Delta sync completed. Downloaded {Count} new messages for {Name}", downloadedMessageIds.Count, Account.Name);
return new DeltaSyncResult(downloadedMessageIds, RequiresFullResync: false);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound ||
(int)ex.HttpStatusCode == 410) // Gone - history expired
{
// History ID is no longer valid (expired or not found)
// This happens when:
// 1. The history ID is too old (Gmail keeps history for ~30 days)
// 2. The account was reset or history was cleared
// Reset the sync identifier and signal that a full resync is needed
_logger.Warning("History ID {HistoryId} expired or not found for {Name}. Full resync required. Error: {Error}",
Account.SynchronizationDeltaIdentifier, Account.Name, ex.Message);
// Clear the sync identifier to trigger initial sync
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor
.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, null)
.ConfigureAwait(false);
return new DeltaSyncResult(downloadedMessageIds, RequiresFullResync: true);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.TooManyRequests)
{
_logger.Warning("Rate limit exceeded during delta sync for {Name}. Retrying after delay.", Account.Name);
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
throw;
}
}
protected override async Task SynchronizeCalendarEventsInternalAsync(CalendarSynchronizationOptions options, CancellationToken cancellationToken = default)
{
_logger.Information("Internal calendar synchronization started for {Name}", Account.Name);
cancellationToken.ThrowIfCancellationRequested();
await SynchronizeCalendarsAsync(cancellationToken).ConfigureAwait(false);
if (options?.Type == CalendarSynchronizationType.CalendarMetadata)
return CalendarSynchronizationResult.Empty;
bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier);
_logger.Debug("Is initial synchronization: {IsInitialSync}", isInitialSync);
var localCalendars = (await _gmailChangeProcessor.GetAccountCalendarsAsync(Account.Id).ConfigureAwait(false))
.Where(c => c.IsSynchronizationEnabled)
.ToList();
var totalCalendars = localCalendars.Count;
if (totalCalendars > 0)
{
UpdateSyncProgress(totalCalendars, totalCalendars, Translator.SyncAction_SynchronizingCalendarEvents);
}
for (int i = 0; i < totalCalendars; i++)
{
var calendar = localCalendars[i];
try
{
var request = _calendarService.Events.List(calendar.RemoteCalendarId);
// Fetch individual event instances (including recurring event occurrences)
// rather than recurring event masters. This ensures we get all occurrences
// as separate events that can be stored and displayed directly.
request.SingleEvents = true;
request.ShowDeleted = true;
if (!string.IsNullOrEmpty(calendar.SynchronizationDeltaToken))
{
request.SyncToken = calendar.SynchronizationDeltaToken;
}
else
{
request.TimeMinDateTimeOffset = DateTimeOffset.UtcNow.AddYears(-1);
}
string nextPageToken;
string syncToken;
var allEvents = new List();
do
{
var events = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (events.Items != null)
{
allEvents.AddRange(events.Items);
}
nextPageToken = events.NextPageToken;
syncToken = events.NextSyncToken;
request.PageToken = nextPageToken;
}
while (!string.IsNullOrEmpty(nextPageToken));
calendar.SynchronizationDeltaToken = syncToken;
var eventByRemoteId = allEvents
.Where(e => !string.IsNullOrWhiteSpace(e.Id))
.GroupBy(e => e.Id, StringComparer.Ordinal)
.ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal);
foreach (var @event in OrderCalendarEventsForPersistence(allEvents))
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await EnsureRecurringParentProcessedAsync(calendar, @event, eventByRemoteId, cancellationToken).ConfigureAwait(false);
await _gmailChangeProcessor.ManageCalendarEventAsync(@event, calendar, Account).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
var errorContext = new SynchronizerErrorContext
{
Account = Account,
ErrorMessage = ex.Message,
Exception = ex,
CalendarId = calendar.Id,
CalendarName = calendar.Name,
OperationType = "CalendarEventSync",
Severity = SynchronizerErrorSeverity.Recoverable
};
_ = await _gmailSynchronizerErrorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
CaptureSynchronizationIssue(errorContext);
_logger.Error(ex, "Failed to process Gmail event {EventId} for calendar {CalendarName}", @event.Id, calendar.Name);
}
}
await _gmailChangeProcessor.UpdateAccountCalendarAsync(calendar).ConfigureAwait(false);
UpdateSyncProgress(totalCalendars, totalCalendars - (i + 1), Translator.SyncAction_SynchronizingCalendarEvents);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
var errorContext = new SynchronizerErrorContext
{
Account = Account,
ErrorMessage = ex.Message,
Exception = ex,
CalendarId = calendar.Id,
CalendarName = calendar.Name,
OperationType = "CalendarSync"
};
_ = await _gmailSynchronizerErrorHandlerFactory.HandleErrorAsync(errorContext).ConfigureAwait(false);
CaptureSynchronizationIssue(errorContext);
if (!errorContext.CanContinueSync)
throw;
UpdateSyncProgress(totalCalendars, totalCalendars - (i + 1), Translator.SyncAction_SynchronizingCalendarEvents);
}
}
return CalendarSynchronizationResult.Empty;
}
private static IEnumerable OrderCalendarEventsForPersistence(IEnumerable events)
=> events
.OrderBy(e => !string.IsNullOrWhiteSpace(e.RecurringEventId))
.ThenByDescending(e => !string.IsNullOrWhiteSpace(GoogleIntegratorExtensions.GetRecurrenceString(e)))
.ThenBy(e => GoogleIntegratorExtensions.GetEventDateTimeOffset(e.Start) ?? DateTimeOffset.MinValue);
private async Task EnsureRecurringParentProcessedAsync(
AccountCalendar calendar,
Event calendarEvent,
Dictionary eventByRemoteId,
CancellationToken cancellationToken)
{
var recurringEventId = calendarEvent?.RecurringEventId;
if (string.IsNullOrWhiteSpace(recurringEventId))
return;
var parentItem = await _gmailChangeProcessor.GetCalendarItemAsync(calendar.Id, recurringEventId).ConfigureAwait(false);
if (parentItem != null)
return;
if (!eventByRemoteId.TryGetValue(recurringEventId, out var parentEvent))
{
try
{
parentEvent = await _calendarService.Events.Get(calendar.RemoteCalendarId, recurringEventId)
.ExecuteAsync(cancellationToken)
.ConfigureAwait(false);
}
catch (GoogleApiException ex)
{
_logger.Warning(ex,
"Failed to fetch recurring parent {ParentRemoteEventId} for child {ChildRemoteEventId} in calendar {CalendarName}",
recurringEventId,
calendarEvent.Id,
calendar.Name);
}
if (parentEvent != null && !string.IsNullOrWhiteSpace(parentEvent.Id))
{
eventByRemoteId[parentEvent.Id] = parentEvent;
}
}
if (parentEvent == null)
{
_logger.Warning(
"Recurring parent {ParentRemoteEventId} is still missing for child {ChildRemoteEventId} in calendar {CalendarName}",
recurringEventId,
calendarEvent.Id,
calendar.Name);
return;
}
await _gmailChangeProcessor.ManageCalendarEventAsync(parentEvent, calendar, Account).ConfigureAwait(false);
}
private async Task SynchronizeCalendarsAsync(CancellationToken cancellationToken = default)
{
var calendarListRequest = _calendarService.CalendarList.List();
var calendarListResponse = await calendarListRequest.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (calendarListResponse.Items == null)
{
_logger.Warning("No calendars found for {Name}", Account.Name);
return;
}
var localCalendars = await _gmailChangeProcessor.GetAccountCalendarsAsync(Account.Id).ConfigureAwait(false);
var remotePrimaryCalendarId = GetPrimaryCalendarId(calendarListResponse.Items);
var usedCalendarColors = new HashSet(StringComparer.OrdinalIgnoreCase);
List insertedCalendars = new();
List updatedCalendars = new();
List deletedCalendars = new();
// 1. Handle deleted calendars.
foreach (var calendar in localCalendars)
{
var remoteCalendar = calendarListResponse.Items.FirstOrDefault(a => a.Id == calendar.RemoteCalendarId);
if (remoteCalendar == null)
{
// Local calendar doesn't exists remotely. Delete local copy.
await _gmailChangeProcessor.DeleteAccountCalendarAsync(calendar).ConfigureAwait(false);
deletedCalendars.Add(calendar);
}
}
// Delete the deleted folders from local list.
deletedCalendars.ForEach(a => localCalendars.Remove(a));
// 2. Handle update/insert based on remote calendars.
foreach (var calendar in calendarListResponse.Items)
{
var existingLocalCalendar = localCalendars.FirstOrDefault(a => a.RemoteCalendarId == calendar.Id);
if (existingLocalCalendar == null)
{
// Insert new calendar.
var remoteBackgroundColor = GetRemoteGmailCalendarBackgroundColor(calendar);
var fallbackColor = ColorHelpers.GetDistinctFlatColorHex(usedCalendarColors, remoteBackgroundColor);
var localCalendar = calendar.AsCalendar(Account.Id, fallbackColor);
localCalendar.IsPrimary = string.Equals(localCalendar.RemoteCalendarId, remotePrimaryCalendarId, StringComparison.OrdinalIgnoreCase);
localCalendar.BackgroundColorHex = ResolveSynchronizedCalendarBackgroundColor(remoteBackgroundColor, localCalendar, usedCalendarColors);
localCalendar.TextColorHex = ColorHelpers.GetReadableTextColorHex(localCalendar.BackgroundColorHex);
usedCalendarColors.Add(localCalendar.BackgroundColorHex);
insertedCalendars.Add(localCalendar);
}
else
{
// Update existing calendar. Right now we only update the name.
var resolvedColor = ResolveSynchronizedCalendarBackgroundColor(GetRemoteGmailCalendarBackgroundColor(calendar), existingLocalCalendar, usedCalendarColors);
if (ShouldUpdateCalendar(calendar, existingLocalCalendar, remotePrimaryCalendarId) ||
!string.Equals(existingLocalCalendar.BackgroundColorHex, resolvedColor, StringComparison.OrdinalIgnoreCase))
{
existingLocalCalendar.Name = calendar.Summary;
existingLocalCalendar.TimeZone = calendar.TimeZone;
existingLocalCalendar.BackgroundColorHex = resolvedColor;
existingLocalCalendar.TextColorHex = ColorHelpers.GetReadableTextColorHex(existingLocalCalendar.BackgroundColorHex);
existingLocalCalendar.IsPrimary = string.Equals(existingLocalCalendar.RemoteCalendarId, remotePrimaryCalendarId, StringComparison.OrdinalIgnoreCase);
existingLocalCalendar.IsReadOnly = !string.Equals(calendar.AccessRole, "owner", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(calendar.AccessRole, "writer", StringComparison.OrdinalIgnoreCase);
updatedCalendars.Add(existingLocalCalendar);
}
else
{
// Remove it from the local folder list to skip additional calendar updates.
localCalendars.Remove(existingLocalCalendar);
}
usedCalendarColors.Add(resolvedColor);
}
}
// 3.Process changes in order-> Insert, Update. Deleted ones are already processed.
foreach (var calendar in insertedCalendars)
{
await _gmailChangeProcessor.InsertAccountCalendarAsync(calendar).ConfigureAwait(false);
}
foreach (var calendar in updatedCalendars)
{
await _gmailChangeProcessor.UpdateAccountCalendarAsync(calendar).ConfigureAwait(false);
}
if (insertedCalendars.Any() || deletedCalendars.Any() || updatedCalendars.Any())
{
// TODO: Notify calendar updates.
// WeakReferenceMessenger.Default.Send(new AccountFolderConfigurationUpdated(Account.Id));
}
}
private async Task InitializeArchiveFolderAsync()
{
var localFolders = await _gmailChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
// Handling of Gmail special virtual Archive folder.
// We will generate a new virtual folder if doesn't exist.
if (!localFolders.Any(a => a.SpecialFolderType == SpecialFolderType.Archive && a.RemoteFolderId == ServiceConstants.ARCHIVE_LABEL_ID))
{
archiveFolderId = Guid.NewGuid();
var archiveFolder = new MailItemFolder()
{
FolderName = "Archive", // will be localized. N/A
RemoteFolderId = ServiceConstants.ARCHIVE_LABEL_ID,
Id = archiveFolderId.Value,
MailAccountId = Account.Id,
SpecialFolderType = SpecialFolderType.Archive,
IsSynchronizationEnabled = true,
IsSystemFolder = true,
IsSticky = true,
IsHidden = false,
ShowUnreadCount = true
};
await _gmailChangeProcessor.InsertFolderAsync(archiveFolder).ConfigureAwait(false);
_isFolderStructureChanged = true;
// Migration-> User might've already have another special folder for Archive.
// We must remove that type assignment.
// This code can be removed after sometime.
var otherArchiveFolders = localFolders.Where(a => a.SpecialFolderType == SpecialFolderType.Archive && a.Id != archiveFolderId.Value).ToList();
if (otherArchiveFolders.Any())
{
_isFolderStructureChanged = true;
}
foreach (var otherArchiveFolder in otherArchiveFolders)
{
otherArchiveFolder.SpecialFolderType = SpecialFolderType.Other;
await _gmailChangeProcessor.UpdateFolderAsync(otherArchiveFolder).ConfigureAwait(false);
}
}
else
{
archiveFolderId = localFolders.First(a => a.SpecialFolderType == SpecialFolderType.Archive && a.RemoteFolderId == ServiceConstants.ARCHIVE_LABEL_ID).Id;
}
}
private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default)
{
var localFolders = await _gmailChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
var folderRequest = _gmailService.Users.Labels.List("me");
var labelsResponse = await folderRequest.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (labelsResponse.Labels == null)
{
_logger.Warning("No folders found for {Name}", Account.Name);
return;
}
List insertedFolders = new();
List updatedFolders = new();
List deletedFolders = new();
// 1. Handle deleted labels.
foreach (var localFolder in localFolders)
{
// Category folder is virtual folder for Wino. Skip it.
if (localFolder.SpecialFolderType == SpecialFolderType.Category) continue;
// Gmail's Archive folder is virtual older for Wino. Skip it.
if (localFolder.SpecialFolderType == SpecialFolderType.Archive) continue;
var remoteFolder = labelsResponse.Labels.FirstOrDefault(a => a.Id == localFolder.RemoteFolderId);
if (remoteFolder == null)
{
// Local folder doesn't exists remotely. Delete local copy.
await _gmailChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false);
deletedFolders.Add(localFolder);
}
}
// Delete the deleted folders from local list.
deletedFolders.ForEach(a => localFolders.Remove(a));
// 2. Handle update/insert based on remote folders.
foreach (var remoteFolder in labelsResponse.Labels)
{
var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.Id);
if (existingLocalFolder == null)
{
// Insert new folder.
var localFolder = remoteFolder.GetLocalFolder(labelsResponse, Account.Id);
insertedFolders.Add(localFolder);
}
else
{
// Update existing folder. Right now we only update the name.
// TODO: Moving folders around different parents. This is not supported right now.
// We will need more comphrensive folder update mechanism to support this.
if (ShouldUpdateFolder(remoteFolder, existingLocalFolder))
{
existingLocalFolder.FolderName = GoogleIntegratorExtensions.GetFolderName(remoteFolder.Name);
existingLocalFolder.TextColorHex = remoteFolder.Color?.TextColor;
existingLocalFolder.BackgroundColorHex = remoteFolder.Color?.BackgroundColor;
updatedFolders.Add(existingLocalFolder);
}
else
{
// Remove it from the local folder list to skip additional folder updates.
localFolders.Remove(existingLocalFolder);
}
}
}
// 3.Process changes in order-> Insert, Update. Deleted ones are already processed.
foreach (var folder in insertedFolders)
{
await _gmailChangeProcessor.InsertFolderAsync(folder).ConfigureAwait(false);
}
foreach (var folder in updatedFolders)
{
await _gmailChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false);
}
if (insertedFolders.Any() || deletedFolders.Any() || updatedFolders.Any())
{
_isFolderStructureChanged = true;
}
}
private bool ShouldUpdateCalendar(CalendarListEntry calendarListEntry, AccountCalendar accountCalendar, string remotePrimaryCalendarId)
{
var remoteCalendarName = calendarListEntry.Summary;
var remoteTimeZone = calendarListEntry.TimeZone;
var remoteBackgroundColor = ResolveSynchronizedCalendarBackgroundColor(GetRemoteGmailCalendarBackgroundColor(calendarListEntry), accountCalendar);
var remoteTextColor = ColorHelpers.GetReadableTextColorHex(remoteBackgroundColor);
var remoteIsPrimary = string.Equals(calendarListEntry.Id, remotePrimaryCalendarId, StringComparison.OrdinalIgnoreCase);
var remoteIsReadOnly = !string.Equals(calendarListEntry.AccessRole, "owner", StringComparison.OrdinalIgnoreCase)
&& !string.Equals(calendarListEntry.AccessRole, "writer", StringComparison.OrdinalIgnoreCase);
bool isNameChanged = !string.Equals(accountCalendar.Name, remoteCalendarName, StringComparison.OrdinalIgnoreCase);
bool isTimeZoneChanged = !string.Equals(accountCalendar.TimeZone, remoteTimeZone, StringComparison.OrdinalIgnoreCase);
bool isBackgroundColorChanged = !string.Equals(accountCalendar.BackgroundColorHex, remoteBackgroundColor, StringComparison.OrdinalIgnoreCase);
bool isTextColorChanged = !string.Equals(accountCalendar.TextColorHex, remoteTextColor, StringComparison.OrdinalIgnoreCase);
bool isPrimaryChanged = accountCalendar.IsPrimary != remoteIsPrimary;
bool isReadOnlyChanged = accountCalendar.IsReadOnly != remoteIsReadOnly;
return isNameChanged || isTimeZoneChanged || isBackgroundColorChanged || isTextColorChanged || isPrimaryChanged || isReadOnlyChanged;
}
private static string GetRemoteGmailCalendarBackgroundColor(CalendarListEntry calendarListEntry)
=> string.IsNullOrWhiteSpace(calendarListEntry?.BackgroundColor) ? null : calendarListEntry.BackgroundColor;
private static string ResolveSynchronizedCalendarBackgroundColor(
string remoteBackgroundColor,
AccountCalendar accountCalendar,
ISet usedCalendarColors = null)
{
if (accountCalendar.IsBackgroundColorUserOverridden)
return accountCalendar.BackgroundColorHex;
var preferredColor = string.IsNullOrWhiteSpace(remoteBackgroundColor)
? accountCalendar.BackgroundColorHex
: remoteBackgroundColor;
return string.IsNullOrWhiteSpace(remoteBackgroundColor) && usedCalendarColors != null
? ColorHelpers.GetDistinctFlatColorHex(usedCalendarColors, preferredColor)
: preferredColor;
}
private string GetPrimaryCalendarId(IList remoteCalendars)
{
if (remoteCalendars == null || remoteCalendars.Count == 0)
return string.Empty;
var explicitPrimary = remoteCalendars.FirstOrDefault(c => c.Primary.GetValueOrDefault());
if (explicitPrimary != null)
return explicitPrimary.Id;
var byPrimaryKeyword = remoteCalendars.FirstOrDefault(c => string.Equals(c.Id, "primary", StringComparison.OrdinalIgnoreCase));
if (byPrimaryKeyword != null)
return byPrimaryKeyword.Id;
var byAccountAddress = remoteCalendars.FirstOrDefault(c => string.Equals(c.Id, Account.Address, StringComparison.OrdinalIgnoreCase));
if (byAccountAddress != null)
return byAccountAddress.Id;
return remoteCalendars.First().Id;
}
private bool ShouldUpdateFolder(Label remoteFolder, MailItemFolder existingLocalFolder)
{
var remoteFolderName = GoogleIntegratorExtensions.GetFolderName(remoteFolder.Name);
var localFolderName = existingLocalFolder.FolderName ?? string.Empty;
bool isNameChanged = !localFolderName.Equals(remoteFolderName, StringComparison.Ordinal);
bool isColorChanged = existingLocalFolder.BackgroundColorHex != remoteFolder.Color?.BackgroundColor ||
existingLocalFolder.TextColorHex != remoteFolder.Color?.TextColor;
return isNameChanged || isColorChanged;
}
///
/// Returns a single get request to retrieve the message with the given id.
/// Always uses Metadata format to download only headers and labels - NOT raw MIME content.
/// MIME content is only downloaded when explicitly needed via DownloadMissingMimeMessageAsync.
///
/// Message to download.
/// Get request for message with Metadata format.
private UsersResource.MessagesResource.GetRequest CreateSingleMessageGet(string messageId)
{
var singleRequest = _gmailService.Users.Messages.Get("me", messageId);
// Always use Metadata format for synchronization - this populates Payload.Headers
// but does NOT download the raw MIME content, saving significant bandwidth and time
singleRequest.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Metadata;
return singleRequest;
}
///
/// Returns a single get request to retrieve the message with Raw format (includes MIME).
/// Used during delta sync to download full message content.
///
/// Message to download.
/// Get request for message with Raw format.
private UsersResource.MessagesResource.GetRequest CreateSingleMessageGetRaw(string messageId)
{
var singleRequest = _gmailService.Users.Messages.Get("me", messageId);
// Use Raw format to get full MIME content
singleRequest.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Raw;
return singleRequest;
}
///
/// Processes the delta changes for the given history changes.
/// Message downloads are not handled here since it's better to batch them.
///
/// List of history changes.
private async Task ProcessHistoryChangesAsync(ListHistoryResponse listHistoryResponse)
{
_logger.Debug("Processing delta change {HistoryId} for {Name}", listHistoryResponse.HistoryId.GetValueOrDefault(), Account.Name);
var pendingStateUpdates = new List();
var pendingAssignmentCreates = new Dictionary(StringComparer.Ordinal);
var pendingAssignmentDeletes = new Dictionary(StringComparer.Ordinal);
var deletedMessageIds = new HashSet(StringComparer.Ordinal);
foreach (var history in listHistoryResponse.History)
{
// Handle label additions.
if (history.LabelsAdded is not null)
{
foreach (var addedLabel in history.LabelsAdded)
{
await HandleLabelAssignmentAsync(addedLabel, pendingStateUpdates, pendingAssignmentCreates, pendingAssignmentDeletes).ConfigureAwait(false);
}
}
// Handle label removals.
if (history.LabelsRemoved is not null)
{
foreach (var removedLabel in history.LabelsRemoved)
{
await HandleLabelRemovalAsync(removedLabel, pendingStateUpdates, pendingAssignmentCreates, pendingAssignmentDeletes).ConfigureAwait(false);
}
}
// Handle removed messages.
if (history.MessagesDeleted is not null)
{
foreach (var deletedMessage in history.MessagesDeleted)
{
var messageId = deletedMessage.Message.Id;
_logger.Debug("Processing message deletion for {MessageId}", messageId);
deletedMessageIds.Add(messageId);
}
}
}
if (pendingStateUpdates.Count > 0)
{
await _gmailChangeProcessor.ApplyMailStateUpdatesAsync(pendingStateUpdates).ConfigureAwait(false);
}
if (pendingAssignmentCreates.Count > 0)
{
await _gmailChangeProcessor.CreateAssignmentsAsync(Account.Id, pendingAssignmentCreates.Values.ToList()).ConfigureAwait(false);
}
if (pendingAssignmentDeletes.Count > 0)
{
await _gmailChangeProcessor.DeleteAssignmentsAsync(Account.Id, pendingAssignmentDeletes.Values.ToList()).ConfigureAwait(false);
}
if (deletedMessageIds.Count > 0)
{
await _gmailChangeProcessor.DeleteMailsAsync(Account.Id, deletedMessageIds).ConfigureAwait(false);
}
}
private static string GetAssignmentChangeKey(string messageId, string labelId)
=> $"{messageId}\u001f{labelId}";
private static void QueueAssignmentChange(
Dictionary creates,
Dictionary deletes,
MailFolderAssignmentUpdate assignment,
bool shouldCreate)
{
if (assignment == null ||
string.IsNullOrWhiteSpace(assignment.MailCopyId) ||
string.IsNullOrWhiteSpace(assignment.RemoteFolderId))
{
return;
}
var key = GetAssignmentChangeKey(assignment.MailCopyId, assignment.RemoteFolderId);
if (shouldCreate)
{
deletes.Remove(key);
creates[key] = assignment;
}
else
{
creates.Remove(key);
deletes[key] = assignment;
}
}
private async Task HandleArchiveAssignmentAsync(
string archivedMessageId,
Dictionary pendingAssignmentCreates,
Dictionary pendingAssignmentDeletes)
{
if (!archiveFolderId.HasValue)
return;
// Ignore if the message is already in the archive.
bool archived = await _gmailChangeProcessor.IsMailExistsInFolderAsync(archivedMessageId, archiveFolderId.Value).ConfigureAwait(false);
if (archived) return;
_logger.Debug("Processing archive assignment for message {Id}", archivedMessageId);
QueueAssignmentChange(
pendingAssignmentCreates,
pendingAssignmentDeletes,
new MailFolderAssignmentUpdate(archivedMessageId, ServiceConstants.ARCHIVE_LABEL_ID),
shouldCreate: true);
}
private async Task HandleUnarchiveAssignmentAsync(
string unarchivedMessageId,
Dictionary pendingAssignmentCreates,
Dictionary pendingAssignmentDeletes)
{
if (!archiveFolderId.HasValue)
return;
// Ignore if the message is not in the archive.
bool archived = await _gmailChangeProcessor.IsMailExistsInFolderAsync(unarchivedMessageId, archiveFolderId.Value).ConfigureAwait(false);
if (!archived) return;
_logger.Debug("Processing un-archive assignment for message {Id}", unarchivedMessageId);
QueueAssignmentChange(
pendingAssignmentCreates,
pendingAssignmentDeletes,
new MailFolderAssignmentUpdate(unarchivedMessageId, ServiceConstants.ARCHIVE_LABEL_ID),
shouldCreate: false);
}
private async Task HandleLabelAssignmentAsync(
HistoryLabelAdded addedLabel,
List pendingStateUpdates,
Dictionary pendingAssignmentCreates,
Dictionary pendingAssignmentDeletes)
{
var messageId = addedLabel.Message.Id;
_logger.Debug("Processing label assignment for message {MessageId}", messageId);
foreach (var labelId in addedLabel.LabelIds)
{
// ARCHIVE is a virtual folder - handle it separately
if (labelId == ServiceConstants.ARCHIVE_LABEL_ID)
{
await HandleArchiveAssignmentAsync(messageId, pendingAssignmentCreates, pendingAssignmentDeletes).ConfigureAwait(false);
continue;
}
// When UNREAD label is added mark the message as un-read.
if (labelId == ServiceConstants.UNREAD_LABEL_ID)
pendingStateUpdates.Add(new MailCopyStateUpdate(messageId, IsRead: false));
// When STARRED label is added mark the message as flagged.
if (labelId == ServiceConstants.STARRED_LABEL_ID)
pendingStateUpdates.Add(new MailCopyStateUpdate(messageId, IsFlagged: true));
QueueAssignmentChange(
pendingAssignmentCreates,
pendingAssignmentDeletes,
new MailFolderAssignmentUpdate(messageId, labelId),
shouldCreate: true);
}
}
private async Task HandleLabelRemovalAsync(
HistoryLabelRemoved removedLabel,
List pendingStateUpdates,
Dictionary pendingAssignmentCreates,
Dictionary pendingAssignmentDeletes)
{
var messageId = removedLabel.Message.Id;
_logger.Debug("Processing label removed for message {MessageId}", messageId);
foreach (var labelId in removedLabel.LabelIds)
{
// ARCHIVE is a virtual folder - handle it separately
if (labelId == ServiceConstants.ARCHIVE_LABEL_ID)
{
await HandleUnarchiveAssignmentAsync(messageId, pendingAssignmentCreates, pendingAssignmentDeletes).ConfigureAwait(false);
continue;
}
// When UNREAD label is removed mark the message as read.
if (labelId == ServiceConstants.UNREAD_LABEL_ID)
pendingStateUpdates.Add(new MailCopyStateUpdate(messageId, IsRead: true));
// When STARRED label is removed mark the message as un-flagged.
if (labelId == ServiceConstants.STARRED_LABEL_ID)
pendingStateUpdates.Add(new MailCopyStateUpdate(messageId, IsFlagged: false));
QueueAssignmentChange(
pendingAssignmentCreates,
pendingAssignmentDeletes,
new MailFolderAssignmentUpdate(messageId, labelId),
shouldCreate: false);
}
}
///
/// Prepares Gmail Draft object from Google SDK.
/// If provided, ThreadId ties the draft to a thread. Used when replying messages.
/// If provided, DraftId updates the draft instead of creating a new one.
///
/// MailKit MimeMessage to include as raw message into Gmail request.
/// ThreadId that this draft should be tied to.
/// Existing DraftId from Gmail to update existing draft.
///
private Draft PrepareGmailDraft(MimeMessage mimeMessage, string messageThreadId = "", string messageDraftId = "")
{
mimeMessage.Prepare(EncodingConstraint.None);
var mimeString = mimeMessage.ToString();
var base64UrlEncodedMime = Base64UrlEncoder.Encode(mimeString);
var nativeMessage = new Message()
{
Raw = base64UrlEncodedMime,
};
if (!string.IsNullOrEmpty(messageThreadId))
nativeMessage.ThreadId = messageThreadId;
var draft = new Draft()
{
Message = nativeMessage,
Id = messageDraftId
};
return draft;
}
#region Mail Integrations
public override List> Move(BatchMoveRequest request)
{
var toFolder = request[0].ToFolder;
var fromFolder = request[0].FromFolder;
// Sent label can't be removed from mails for Gmail.
// They are automatically assigned by Gmail.
// When you delete sent mail from gmail web portal, it's moved to Trash
// but still has Sent label. It's just hidden from the user.
// Proper assignments will be done later on CreateAssignment call to mimic this behavior.
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList(),
AddLabelIds = [toFolder.RemoteFolderId]
};
// Archived item is being moved to different folder.
// Unarchive will move it to Inbox, so this is a different case.
// We can't remove ARCHIVE label because it's a virtual folder and does not exist in Gmail.
// We will just add the target label and Gmail will handle the rest.
if (fromFolder.SpecialFolderType == SpecialFolderType.Archive)
{
batchModifyRequest.AddLabelIds = [toFolder.RemoteFolderId];
}
else if (fromFolder.SpecialFolderType != SpecialFolderType.Sent)
{
// Only add remove label ids if the source folder is not sent folder.
batchModifyRequest.RemoveLabelIds = [fromFolder.RemoteFolderId];
}
var networkCall = _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> ChangeFlag(BatchChangeFlagRequest request)
{
bool isFlagged = request[0].IsFlagged;
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList(),
};
if (isFlagged)
batchModifyRequest.AddLabelIds = new List() { ServiceConstants.STARRED_LABEL_ID };
else
batchModifyRequest.RemoveLabelIds = new List() { ServiceConstants.STARRED_LABEL_ID };
var networkCall = _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> ChangeJunkState(BatchChangeJunkStateRequest request)
{
bool isJunk = request[0].IsJunk;
var addLabelIds = new HashSet();
var removeLabelIds = new HashSet();
if (isJunk)
{
addLabelIds.Add(ServiceConstants.SPAM_LABEL_ID);
removeLabelIds.Add(ServiceConstants.INBOX_LABEL_ID);
}
else
{
addLabelIds.Add(ServiceConstants.INBOX_LABEL_ID);
removeLabelIds.Add(ServiceConstants.SPAM_LABEL_ID);
}
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList(),
AddLabelIds = addLabelIds.ToList(),
RemoveLabelIds = removeLabelIds.ToList()
};
var networkCall = _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> MarkRead(BatchMarkReadRequest request)
{
bool readStatus = request[0].IsRead;
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList(),
};
if (readStatus)
batchModifyRequest.RemoveLabelIds = new List() { ServiceConstants.UNREAD_LABEL_ID };
else
batchModifyRequest.AddLabelIds = new List() { ServiceConstants.UNREAD_LABEL_ID };
var networkCall = _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> Delete(BatchDeleteRequest request)
{
var batchModifyRequest = new BatchDeleteMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList(),
};
var networkCall = _gmailService.Users.Messages.BatchDelete(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> CreateDraft(CreateDraftRequest singleRequest)
{
Draft draft = null;
// It's new mail. Not a reply
if (singleRequest.DraftPreperationRequest.ReferenceMailCopy == null)
draft = PrepareGmailDraft(singleRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage);
else
draft = PrepareGmailDraft(singleRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage,
singleRequest.DraftPreperationRequest.ReferenceMailCopy.ThreadId,
singleRequest.DraftPreperationRequest.ReferenceMailCopy.DraftId);
var networkCall = _gmailService.Users.Drafts.Create(draft, "me");
return [new HttpRequestBundle(networkCall, singleRequest, singleRequest)];
}
public override List> Archive(BatchArchiveRequest request)
{
bool isArchiving = request[0].IsArchiving;
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = request.Select(a => a.Item.Id.ToString()).ToList()
};
if (isArchiving)
{
batchModifyRequest.RemoveLabelIds = new[] { ServiceConstants.INBOX_LABEL_ID };
}
else
{
batchModifyRequest.AddLabelIds = new[] { ServiceConstants.INBOX_LABEL_ID };
}
var networkCall = _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
return [new HttpRequestBundle(networkCall, request)];
}
public override List> SendDraft(SendDraftRequest singleDraftRequest)
{
var message = new Message();
if (!string.IsNullOrEmpty(singleDraftRequest.Item.ThreadId))
{
message.ThreadId = singleDraftRequest.Item.ThreadId;
}
// Local draft mapping header must never leak to recipients.
singleDraftRequest.Request.Mime.Headers.Remove(Domain.Constants.WinoLocalDraftHeader);
singleDraftRequest.Request.Mime.Prepare(EncodingConstraint.None);
var mimeString = singleDraftRequest.Request.Mime.ToString();
var base64UrlEncodedMime = Base64UrlEncoder.Encode(mimeString);
message.Raw = base64UrlEncodedMime;
var draft = new Draft()
{
Id = singleDraftRequest.Request.MailItem.DraftId,
Message = message
};
var networkCall = _gmailService.Users.Drafts.Send(draft, "me");
return [new HttpRequestBundle(networkCall, singleDraftRequest, singleDraftRequest)];
}
public override async Task> OnlineSearchAsync(string queryText, List folders, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(queryText))
return [];
static bool IsArchiveFolder(IMailItemFolder folder)
=> folder?.SpecialFolderType == SpecialFolderType.Archive || folder?.RemoteFolderId == ServiceConstants.ARCHIVE_LABEL_ID;
var distinctFolders = folders?
.Where(folder => folder != null)
.GroupBy(folder => folder.Id)
.Select(group => group.First())
.ToList();
var messageIds = new HashSet(StringComparer.Ordinal);
async Task CollectMessageIdsAsync(UsersResource.MessagesResource.ListRequest request)
{
string pageToken = null;
do
{
if (!string.IsNullOrEmpty(pageToken))
{
request.PageToken = pageToken;
}
var response = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (response.Messages == null || response.Messages.Count == 0) break;
foreach (var message in response.Messages)
{
if (!string.IsNullOrEmpty(message.Id))
{
messageIds.Add(message.Id);
}
}
pageToken = response.NextPageToken;
} while (!string.IsNullOrEmpty(pageToken));
}
bool hasScopedQuery = queryText.StartsWith("label:", StringComparison.OrdinalIgnoreCase) ||
queryText.StartsWith("in:", StringComparison.OrdinalIgnoreCase);
if (hasScopedQuery || distinctFolders?.Count == 0)
{
var request = _gmailService.Users.Messages.List("me");
request.Q = queryText;
request.MaxResults = 500;
await CollectMessageIdsAsync(request).ConfigureAwait(false);
}
else
{
foreach (var folder in distinctFolders)
{
cancellationToken.ThrowIfCancellationRequested();
var request = _gmailService.Users.Messages.List("me");
request.MaxResults = 500;
if (IsArchiveFolder(folder))
{
// Gmail archive is virtual. Query via search operator instead of label id.
request.Q = $"in:archive {queryText}".Trim();
}
else
{
request.Q = queryText;
request.LabelIds = new List { folder.RemoteFolderId };
}
await CollectMessageIdsAsync(request).ConfigureAwait(false);
}
}
if (messageIds.Count == 0)
return [];
var messageIdList = messageIds.ToList();
// Do not download messages that already exist locally.
var existingMessageIds = await _gmailChangeProcessor.AreMailsExistsAsync(messageIdList).ConfigureAwait(false);
var messagesToDownload = messageIdList.Except(existingMessageIds, StringComparer.Ordinal);
// Download missing messages in batch with metadata only.
await DownloadMessagesInBatchAsync(messagesToDownload, cancellationToken).ConfigureAwait(false);
// Get results from database and return.
return await _gmailChangeProcessor.GetMailCopiesAsync(messageIdList).ConfigureAwait(false);
}
///
/// Downloads multiple messages in batches with metadata only (no MIME) and creates mail packages.
/// Uses Gmail batch API to download up to MaximumAllowedBatchRequestSize messages per request.
/// Used for initial sync where MIME is not needed.
///
/// List of Gmail message IDs to download
/// Cancellation token
private async Task DownloadMessagesInBatchAsync(IEnumerable messageIds, CancellationToken cancellationToken = default)
{
await DownloadMessagesInBatchAsync(messageIds, downloadRawMime: false, cancellationToken).ConfigureAwait(false);
}
///
/// Downloads multiple messages in batches with optional MIME content and creates mail packages.
/// Uses Gmail batch API to download up to MaximumAllowedBatchRequestSize messages per request.
///
/// List of Gmail message IDs to download
/// True to download Raw format with MIME, false for Metadata only
/// Cancellation token
private async Task DownloadMessagesInBatchAsync(IEnumerable messageIds, bool downloadRawMime, CancellationToken cancellationToken = default)
{
var messageIdList = messageIds.ToList();
if (messageIdList.Count == 0) return;
// Split into batches based on MaximumAllowedBatchRequestSize
var batches = messageIdList.Batch((int)MaximumAllowedBatchRequestSize);
foreach (var batch in batches)
{
var batchRequest = new BatchRequest(_gmailService);
var downloadedMessages = new List();
var batchTasks = new List();
foreach (var messageId in batch)
{
var request = downloadRawMime ? CreateSingleMessageGetRaw(messageId) : CreateSingleMessageGet(messageId);
batchRequest.Queue(request, (message, error, index, httpMessage) =>
{
var task = Task.Run(async () =>
{
if (error != null)
{
_logger.Warning("Failed to download message {MessageId}: {Error}", messageId, error.Message);
return;
}
if (message != null)
{
lock (downloadedMessages)
{
downloadedMessages.Add(message);
}
}
});
batchTasks.Add(task);
});
}
// Execute the batch request
await batchRequest.ExecuteAsync(cancellationToken).ConfigureAwait(false);
await Task.WhenAll(batchTasks).ConfigureAwait(false);
// Process all downloaded messages
var pendingPackages = new List();
foreach (var gmailMessage in downloadedMessages)
{
try
{
// Create mail packages from metadata/raw.
// If Gmail response is Raw format, CreateNewMailPackagesAsync will parse MIME and
// include it in package(s) so it can be saved to disk.
var packages = await CreateNewMailPackagesAsync(gmailMessage, null, cancellationToken).ConfigureAwait(false);
if (packages != null)
pendingPackages.AddRange(packages);
// Update sync identifier if available
if (gmailMessage.HistoryId.HasValue)
{
await UpdateAccountSyncIdentifierAsync(gmailMessage.HistoryId.Value).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to process downloaded message {MessageId}", gmailMessage.Id);
}
}
if (pendingPackages.Count > 0)
{
await _gmailChangeProcessor.CreateMailsAsync(Account.Id, pendingPackages).ConfigureAwait(false);
}
}
}
///
/// Downloads a single message by ID with metadata only (no MIME) and creates mail packages.
///
/// Gmail message ID to download
/// Cancellation token
private async Task DownloadSingleMessageMetadataAsync(string messageId, CancellationToken cancellationToken = default)
{
var request = CreateSingleMessageGet(messageId);
var gmailMessage = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (gmailMessage == null)
{
_logger.Warning("Failed to download message metadata for {MessageId}", messageId);
return;
}
// Create mail packages from metadata
var packages = await CreateNewMailPackagesAsync(gmailMessage, null, cancellationToken).ConfigureAwait(false);
if (packages != null && packages.Count > 0)
{
await _gmailChangeProcessor.CreateMailsAsync(Account.Id, packages).ConfigureAwait(false);
}
// Update sync identifier if available
if (gmailMessage.HistoryId.HasValue)
{
await UpdateAccountSyncIdentifierAsync(gmailMessage.HistoryId.Value).ConfigureAwait(false);
}
}
public override async Task DownloadMissingMimeMessageAsync(MailCopy mailItem,
ITransferProgress transferProgress = null,
CancellationToken cancellationToken = default)
{
try
{
var request = _gmailService.Users.Messages.Get("me", mailItem.Id);
request.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Raw;
var gmailMessage = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
var mimeMessage = gmailMessage.GetGmailMimeMessage();
if (mimeMessage == null)
{
_logger.Warning("Tried to download Gmail Raw Mime with {Id} id and server responded without a data.", mailItem.Id);
return;
}
await _gmailChangeProcessor.SaveMimeFileAsync(mailItem.FileId, mimeMessage, Account.Id).ConfigureAwait(false);
}
catch (GoogleApiException ex) when (ex.HttpStatusCode == System.Net.HttpStatusCode.NotFound)
{
_logger.Warning("Gmail message {MailId} not found (404) during MIME download. Deleting locally.", mailItem.Id);
await _gmailChangeProcessor.DeleteMailAsync(Account.Id, mailItem.Id).ConfigureAwait(false);
throw new SynchronizerEntityNotFoundException(ex.Message);
}
}
public override async Task DownloadCalendarAttachmentAsync(
Wino.Core.Domain.Entities.Calendar.CalendarItem calendarItem,
Wino.Core.Domain.Entities.Calendar.CalendarAttachment attachment,
string localFilePath,
CancellationToken cancellationToken = default)
{
try
{
// Gmail calendar attachments are stored in Google Drive
// RemoteAttachmentId contains either FileId or FileUrl
// For simplicity, we'll try to download from the FileId/FileUrl
if (string.IsNullOrEmpty(attachment.RemoteAttachmentId))
{
_logger.Error("RemoteAttachmentId is empty for attachment {AttachmentId}", attachment.Id);
throw new InvalidOperationException("RemoteAttachmentId is required to download Gmail calendar attachment.");
}
// Gmail calendar attachments are links to Google Drive files
// The attachment.RemoteAttachmentId is either a FileId or FileUrl
// Since we can't directly download from Calendar API, this would require Drive API access
// For now, throw NotSupportedException as Gmail attachments require additional Drive API setup
_logger.Warning("Gmail calendar attachment download requires Google Drive API access. FileId/URL: {RemoteId}", attachment.RemoteAttachmentId);
throw new NotSupportedException("Gmail calendar attachments are stored in Google Drive and require additional API configuration to download.");
}
catch (Exception ex)
{
_logger.Error(ex, "Error downloading Gmail calendar attachment {AttachmentId}", attachment.Id);
throw;
}
}
public override List> RenameFolder(RenameFolderRequest request)
{
var label = new Label()
{
Name = request.NewFolderName
};
var networkCall = _gmailService.Users.Labels.Update(label, "me", request.Folder.RemoteFolderId);
return [new HttpRequestBundle(networkCall, request, request)];
}
public override List> EmptyFolder(EmptyFolderRequest request)
{
// Create batch delete request.
var deleteRequests = request.MailsToDelete.Select(a => new DeleteRequest(a));
return Delete(new BatchDeleteRequest(deleteRequests));
}
public override List> MarkFolderAsRead(MarkFolderAsReadRequest request)
=> MarkRead(new BatchMarkReadRequest(request.MailsToMarkRead.Select(a => new MarkReadRequest(a, true))));
public override List> DeleteFolder(DeleteFolderRequest request)
{
var networkCall = _gmailService.Users.Labels.Delete("me", request.Folder.RemoteFolderId);
return [new HttpRequestBundle(networkCall, request, request)];
}
public override List> CreateSubFolder(CreateSubFolderRequest request)
{
var parentLabelName = request.Folder.FolderName;
try
{
var parentLabel = _gmailService.Users.Labels.Get("me", request.Folder.RemoteFolderId).Execute();
if (!string.IsNullOrWhiteSpace(parentLabel?.Name))
{
parentLabelName = parentLabel.Name;
}
}
catch (Exception ex)
{
_logger.Warning(ex, "Failed to resolve full parent label name for {FolderId}. Falling back to local folder name.", request.Folder.RemoteFolderId);
}
var label = new Label()
{
Name = $"{parentLabelName}/{request.NewFolderName}"
};
var networkCall = _gmailService.Users.Labels.Create(label, "me");
return [new HttpRequestBundle(networkCall, request, request)];
}
public override List> CreateRootFolder(CreateRootFolderRequest request)
{
var label = new Label()
{
Name = request.NewFolderName
};
var networkCall = _gmailService.Users.Labels.Create(label, "me");
return [new HttpRequestBundle(networkCall, request, request)];
}
#endregion
#region Request Execution
public override async Task ExecuteNativeRequestsAsync(List> batchedRequests,
CancellationToken cancellationToken = default)
{
// First apply all UI changes immediately before any batching.
// This ensures UI reflects changes right away, regardless of batch processing.
foreach (var bundle in batchedRequests)
{
bundle.UIChangeRequest?.ApplyUIChanges();
}
// Batch requests per Google service instance. Calendar requests must be queued against
// CalendarService, otherwise Gmail's batch endpoint will reject Calendar REST paths.
var requestGroups = batchedRequests.GroupBy(bundle => bundle.NativeRequest.Service);
foreach (var requestGroup in requestGroups)
{
var batchedBundles = requestGroup.Batch((int)MaximumAllowedBatchRequestSize);
foreach (var bundle in batchedBundles)
{
var nativeBatchRequest = new BatchRequest(requestGroup.Key);
var bundleTasks = new List();
foreach (var requestBundle in bundle)
{
// UI changes are already applied above before batching.
nativeBatchRequest.Queue