Cleaning up the solution. Separating Shared.WinRT, Services and Synchronization. Removing synchronization from app. Reducing bundle size by 45mb.

This commit is contained in:
Burak Kaan Köse
2024-07-21 05:45:02 +02:00
parent f112f369a7
commit 495885e006
523 changed files with 2254 additions and 2375 deletions

View File

@@ -0,0 +1,266 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Web;
using Google.Apis.Gmail.v1.Data;
using MimeKit;
using MimeKit.IO;
using MimeKit.IO.Filters;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Constants = Wino.Domain.Constants;
namespace Wino.Core.Extensions
{
public static class GoogleIntegratorExtensions
{
public const string INBOX_LABEL_ID = "INBOX";
public const string UNREAD_LABEL_ID = "UNREAD";
public const string IMPORTANT_LABEL_ID = "IMPORTANT";
public const string STARRED_LABEL_ID = "STARRED";
public const string DRAFT_LABEL_ID = "DRAFT";
public const string SENT_LABEL_ID = "SENT";
public const string SPAM_LABEL_ID = "SPAM";
public const string CHAT_LABEL_ID = "CHAT";
public const string TRASH_LABEL_ID = "TRASH";
// Label visibility identifiers.
private const string SYSTEM_FOLDER_IDENTIFIER = "system";
private const string FOLDER_HIDE_IDENTIFIER = "labelHide";
private const string CATEGORY_PREFIX = "CATEGORY_";
private const string FOLDER_SEPERATOR_STRING = "/";
private const char FOLDER_SEPERATOR_CHAR = '/';
private static Dictionary<string, SpecialFolderType> KnownFolderDictionary = new Dictionary<string, SpecialFolderType>()
{
{ INBOX_LABEL_ID, SpecialFolderType.Inbox },
{ CHAT_LABEL_ID, SpecialFolderType.Chat },
{ IMPORTANT_LABEL_ID, SpecialFolderType.Important },
{ TRASH_LABEL_ID, SpecialFolderType.Deleted },
{ DRAFT_LABEL_ID, SpecialFolderType.Draft },
{ SENT_LABEL_ID, SpecialFolderType.Sent },
{ SPAM_LABEL_ID, SpecialFolderType.Junk },
{ STARRED_LABEL_ID, SpecialFolderType.Starred },
{ UNREAD_LABEL_ID, SpecialFolderType.Unread },
{ Constants.FORUMS_LABEL_ID, SpecialFolderType.Forums },
{ Constants.UPDATES_LABEL_ID, SpecialFolderType.Updates },
{ Constants.PROMOTIONS_LABEL_ID, SpecialFolderType.Promotions },
{ Constants.SOCIAL_LABEL_ID, SpecialFolderType.Social},
{ Constants.PERSONAL_LABEL_ID, SpecialFolderType.Personal},
};
private static string GetNormalizedLabelName(string labelName)
{
// 1. Remove CATEGORY_ prefix.
var normalizedLabelName = labelName.Replace(CATEGORY_PREFIX, string.Empty);
// 2. Normalize label name by capitalizing first letter.
normalizedLabelName = char.ToUpper(normalizedLabelName[0]) + normalizedLabelName.Substring(1).ToLower();
return normalizedLabelName;
}
public static MailItemFolder GetLocalFolder(this Label label, ListLabelsResponse labelsResponse, Guid accountId)
{
bool isAllCapital = label.Name.All(a => char.IsUpper(a));
var normalizedLabelName = GetFolderName(label);
// Even though we normalize the label name, check is done by capitalizing the label name.
var capitalNormalizedLabelName = normalizedLabelName.ToUpper();
bool isSpecialFolder = KnownFolderDictionary.ContainsKey(capitalNormalizedLabelName);
var specialFolderType = isSpecialFolder ? KnownFolderDictionary[capitalNormalizedLabelName] : SpecialFolderType.Other;
// We used to support FOLDER_HIDE_IDENTIFIER to hide invisible folders.
// However, a lot of people complained that they don't see their folders after the initial sync
// without realizing that they are hidden in Gmail settings. Therefore, it makes more sense to ignore Gmail's configuration
// since Wino allows folder visibility configuration separately.
// Overridden hidden labels are shown in the UI, but they have their synchronization disabled.
// This is mainly because 'All Mails' label is hidden by default in Gmail, but there is no point to download all mails.
bool shouldEnableSynchronization = label.LabelListVisibility != FOLDER_HIDE_IDENTIFIER;
bool isHidden = false;
bool isChildOfCategoryFolder = label.Name.StartsWith(CATEGORY_PREFIX);
bool isSticky = isSpecialFolder && specialFolderType != SpecialFolderType.Category && !isChildOfCategoryFolder;
// By default, all special folders update unread count in the UI except Trash.
bool shouldShowUnreadCount = specialFolderType != SpecialFolderType.Deleted || specialFolderType != SpecialFolderType.Other;
bool isSystemFolder = label.Type == SYSTEM_FOLDER_IDENTIFIER;
var localFolder = new MailItemFolder()
{
TextColorHex = label.Color?.TextColor,
BackgroundColorHex = label.Color?.BackgroundColor,
FolderName = normalizedLabelName,
RemoteFolderId = label.Id,
Id = Guid.NewGuid(),
MailAccountId = accountId,
IsSynchronizationEnabled = shouldEnableSynchronization,
SpecialFolderType = specialFolderType,
IsSystemFolder = isSystemFolder,
IsSticky = isSticky,
IsHidden = isHidden,
ShowUnreadCount = shouldShowUnreadCount,
};
localFolder.ParentRemoteFolderId = isChildOfCategoryFolder ? string.Empty : GetParentFolderRemoteId(label.Name, labelsResponse);
return localFolder;
}
public static bool GetIsDraft(this Message message)
=> message?.LabelIds?.Any(a => a == DRAFT_LABEL_ID) ?? false;
public static bool GetIsUnread(this Message message)
=> message?.LabelIds?.Any(a => a == UNREAD_LABEL_ID) ?? false;
public static bool GetIsFocused(this Message message)
=> message?.LabelIds?.Any(a => a == IMPORTANT_LABEL_ID) ?? false;
public static bool GetIsFlagged(this Message message)
=> message?.LabelIds?.Any(a => a == STARRED_LABEL_ID) ?? false;
private static string GetParentFolderRemoteId(string fullLabelName, ListLabelsResponse labelsResponse)
{
if (string.IsNullOrEmpty(fullLabelName)) return string.Empty;
// Find the last index of '/'
int lastIndex = fullLabelName.LastIndexOf('/');
// If '/' not found or it's at the start, return the empty string.
if (lastIndex <= 0) return string.Empty;
// Extract the parent label
var parentLabelName = fullLabelName.Substring(0, lastIndex);
return labelsResponse.Labels.FirstOrDefault(a => a.Name == parentLabelName)?.Id ?? string.Empty;
}
public static string GetFolderName(Label label)
{
if (string.IsNullOrEmpty(label.Name)) return string.Empty;
// Folders with "//" at the end has "/" as the name.
if (label.Name.EndsWith(FOLDER_SEPERATOR_STRING)) return FOLDER_SEPERATOR_STRING;
string[] parts = label.Name.Split(FOLDER_SEPERATOR_CHAR);
var lastPart = parts[parts.Length - 1];
return GetNormalizedLabelName(lastPart);
}
/// <summary>
/// Returns MailCopy out of native Gmail message and converted MimeMessage of that native messaage.
/// </summary>
/// <param name="gmailMessage">Gmail Message</param>
/// <param name="mimeMessage">MimeMessage representation of that native message.</param>
/// <returns>MailCopy object that is ready to be inserted to database.</returns>
public static MailCopy AsMailCopy(this Message gmailMessage, MimeMessage mimeMessage)
{
bool isUnread = gmailMessage.GetIsUnread();
bool isFocused = gmailMessage.GetIsFocused();
bool isFlagged = gmailMessage.GetIsFlagged();
bool isDraft = gmailMessage.GetIsDraft();
return new MailCopy()
{
CreationDate = mimeMessage.Date.UtcDateTime,
Subject = HttpUtility.HtmlDecode(mimeMessage.Subject),
FromName = MailkitClientExtensions.GetActualSenderName(mimeMessage),
FromAddress = MailkitClientExtensions.GetActualSenderAddress(mimeMessage),
PreviewText = HttpUtility.HtmlDecode(gmailMessage.Snippet),
ThreadId = gmailMessage.ThreadId,
Importance = (MailImportance)mimeMessage.Importance,
Id = gmailMessage.Id,
IsDraft = isDraft,
HasAttachments = mimeMessage.Attachments.Any(),
IsRead = !isUnread,
IsFlagged = isFlagged,
IsFocused = isFocused,
InReplyTo = mimeMessage.InReplyTo,
MessageId = mimeMessage.MessageId,
References = mimeMessage.References.GetReferences(),
FileId = Guid.NewGuid()
};
}
public static Tuple<MailCopy, MimeMessage, IEnumerable<string>> GetMailDetails(this Message message)
{
MimeMessage mimeMessage = message.GetGmailMimeMessage();
if (mimeMessage == null)
{
// This should never happen.
Debugger.Break();
return default;
}
bool isUnread = message.GetIsUnread();
bool isFocused = message.GetIsFocused();
bool isFlagged = message.GetIsFlagged();
bool isDraft = message.GetIsDraft();
var mailCopy = new MailCopy()
{
CreationDate = mimeMessage.Date.UtcDateTime,
Subject = HttpUtility.HtmlDecode(mimeMessage.Subject),
FromName = MailkitClientExtensions.GetActualSenderName(mimeMessage),
FromAddress = MailkitClientExtensions.GetActualSenderAddress(mimeMessage),
PreviewText = HttpUtility.HtmlDecode(message.Snippet),
ThreadId = message.ThreadId,
Importance = (MailImportance)mimeMessage.Importance,
Id = message.Id,
IsDraft = isDraft,
HasAttachments = mimeMessage.Attachments.Any(),
IsRead = !isUnread,
IsFlagged = isFlagged,
IsFocused = isFocused,
InReplyTo = mimeMessage.InReplyTo,
MessageId = mimeMessage.MessageId,
References = mimeMessage.References.GetReferences()
};
return new Tuple<MailCopy, MimeMessage, IEnumerable<string>>(mailCopy, mimeMessage, message.LabelIds);
}
/// <summary>
/// Returns MimeKit.MimeMessage instance for this GMail Message's Raw content.
/// </summary>
/// <param name="message">GMail message.</param>
public static MimeMessage GetGmailMimeMessage(this Message message)
{
if (message == null || message.Raw == null)
return null;
// Gmail raw is not base64 but base64Safe. We need to remove this HTML things.
var base64Encoded = message.Raw.Replace(",", "=").Replace("-", "+").Replace("_", "/");
byte[] bytes = Encoding.ASCII.GetBytes(base64Encoded);
var stream = new MemoryStream(bytes);
// This method will dispose outer stream.
using (stream)
{
using var filtered = new FilteredStream(stream);
filtered.Add(DecoderFilter.Create(ContentEncoding.Base64));
return MimeMessage.Load(filtered);
}
}
}
}

View File

@@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Wino.Domain.Enums;
using Wino.Domain.Interfaces;
namespace Wino.Core.Extensions
{
public static class ListExtensions
{
public static IEnumerable<T> FlattenBy<T>(this IEnumerable<T> nodes, Func<T, IEnumerable<T>> selector)
{
if (nodes.Any() == false)
return nodes;
var descendants = nodes
.SelectMany(selector)
.FlattenBy(selector);
return nodes.Concat(descendants);
}
public static IEnumerable<IBatchChangeRequest> CreateBatch(this IEnumerable<IGrouping<MailSynchronizerOperation, IRequestBase>> items)
{
IBatchChangeRequest batch = null;
foreach (var group in items)
{
var key = group.Key;
}
yield return batch;
}
public static void AddSorted<T>(this List<T> @this, T item) where T : IComparable<T>
{
if (@this.Count == 0)
{
@this.Add(item);
return;
}
if (@this[@this.Count - 1].CompareTo(item) <= 0)
{
@this.Add(item);
return;
}
if (@this[0].CompareTo(item) >= 0)
{
@this.Insert(0, item);
return;
}
int index = @this.BinarySearch(item);
if (index < 0)
index = ~index;
@this.Insert(index, item);
}
}
}

View File

@@ -0,0 +1,178 @@
using System;
using System.Linq;
using MailKit;
using MimeKit;
using Wino.Domain;
using Wino.Domain;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Wino.Services.Extensions;
namespace Wino.Core.Extensions
{
public static class MailkitClientExtensions
{
public static string CreateUid(Guid folderId, uint messageUid)
=> $"{folderId}{Constants.MailCopyUidSeparator}{messageUid}";
public static MailImportance GetImportance(this MimeMessage messageSummary)
{
if (messageSummary.Headers != null && messageSummary.Headers.Contains(HeaderId.Importance))
{
var rawImportance = messageSummary.Headers[HeaderId.Importance];
return rawImportance switch
{
"Low" => MailImportance.Low,
"High" => MailImportance.High,
_ => MailImportance.Normal,
};
}
return MailImportance.Normal;
}
public static bool GetIsRead(this MessageFlags? flags)
=> flags.GetValueOrDefault().HasFlag(MessageFlags.Seen);
public static bool GetIsFlagged(this MessageFlags? flags)
=> flags.GetValueOrDefault().HasFlag(MessageFlags.Flagged);
public static string GetThreadId(this IMessageSummary messageSummary)
{
// First check whether we have the default values.
if (!string.IsNullOrEmpty(messageSummary.ThreadId))
return messageSummary.ThreadId;
if (messageSummary.GMailThreadId != null)
return messageSummary.GMailThreadId.ToString();
return default;
}
public static string GetMessageId(this MimeMessage mimeMessage)
=> mimeMessage.MessageId;
public static string GetReferences(this MessageIdList messageIdList)
=> string.Join(";", messageIdList);
public static string GetInReplyTo(this MimeMessage mimeMessage)
{
if (mimeMessage.Headers.Contains(HeaderId.InReplyTo))
{
// Normalize if <> brackets are there.
var inReplyTo = mimeMessage.Headers[HeaderId.InReplyTo];
if (inReplyTo.StartsWith("<") && inReplyTo.EndsWith(">"))
return inReplyTo.Substring(1, inReplyTo.Length - 2);
return inReplyTo;
}
return string.Empty;
}
private static string GetPreviewText(this MimeMessage message)
{
if (string.IsNullOrEmpty(message.HtmlBody))
return message.TextBody;
else
return HtmlAgilityPackExtensions.GetPreviewText(message.HtmlBody);
}
public static MailCopy GetMailDetails(this IMessageSummary messageSummary, MailItemFolder folder, MimeMessage mime)
{
// MessageSummary will only have UniqueId, Flags, ThreadId.
// Other properties are extracted directly from the MimeMessage.
// IMAP doesn't have unique id for mails.
// All mails are mapped to specific folders with incremental Id.
// Uid 1 may belong to different messages in different folders, but can never be
// same for different messages in same folders.
// Here we create arbitrary Id that maps the Id of the message with Folder UniqueId.
// When folder becomes invalid, we'll clear out these MailCopies as well.
var messageUid = CreateUid(folder.Id, messageSummary.UniqueId.Id);
var previewText = mime.GetPreviewText();
var copy = new MailCopy()
{
Id = messageUid,
CreationDate = mime.Date.UtcDateTime,
ThreadId = messageSummary.GetThreadId(),
MessageId = mime.GetMessageId(),
Subject = mime.Subject,
IsRead = messageSummary.Flags.GetIsRead(),
IsFlagged = messageSummary.Flags.GetIsFlagged(),
PreviewText = previewText,
FromAddress = GetActualSenderAddress(mime),
FromName = GetActualSenderName(mime),
IsFocused = false,
Importance = mime.GetImportance(),
References = mime.References?.GetReferences(),
InReplyTo = mime.GetInReplyTo(),
HasAttachments = mime.Attachments.Any(),
FileId = Guid.NewGuid()
};
return copy;
}
// TODO: Name and Address parsing should be handled better.
// At some point Wino needs better contact management.
public static string GetActualSenderName(MimeMessage message)
{
if (message == null)
return string.Empty;
// From MimeKit
// The "From" header specifies the author(s) of the message.
// If more than one MimeKit.MailboxAddress is added to the list of "From" addresses,
// the MimeKit.MimeMessage.Sender should be set to the single MimeKit.MailboxAddress
// of the personal actually sending the message.
// Also handle: https://stackoverflow.com/questions/46474030/mailkit-from-address
if (message.Sender != null)
return string.IsNullOrEmpty(message.Sender.Name) ? message.Sender.Address : message.Sender.Name;
else if (message.From?.Mailboxes != null)
{
var firstAvailableName = message.From.Mailboxes.FirstOrDefault(a => !string.IsNullOrEmpty(a.Name))?.Name;
if (string.IsNullOrEmpty(firstAvailableName))
{
var firstAvailableAddress = message.From.Mailboxes.FirstOrDefault(a => !string.IsNullOrEmpty(a.Address))?.Address;
if (!string.IsNullOrEmpty(firstAvailableAddress))
{
return firstAvailableAddress;
}
}
return firstAvailableName;
}
// No sender, no from, I don't know what to do.
return Translator.UnknownSender;
}
// TODO: This is wrong.
public static string GetActualSenderAddress(MimeMessage mime)
{
if (mime == null)
return string.Empty;
bool hasSingleFromMailbox = mime.From.Mailboxes.Count() == 1;
if (hasSingleFromMailbox)
return mime.From.Mailboxes.First().GetAddress(idnEncode: true);
else if (mime.Sender != null)
return mime.Sender.GetAddress(idnEncode: true);
else
return Translator.UnknownSender;
}
}
}

View File

@@ -0,0 +1,22 @@
using System;
using MailKit;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
namespace Wino.Core.Extensions
{
public static class MailkitExtensions
{
public static MailItemFolder GetLocalFolder(this IMailFolder mailkitMailFolder)
{
return new MailItemFolder()
{
Id = Guid.NewGuid(),
FolderName = mailkitMailFolder.Name,
RemoteFolderId = mailkitMailFolder.FullName,
ParentRemoteFolderId = mailkitMailFolder.ParentFolder?.FullName,
SpecialFolderType = SpecialFolderType.Other
};
}
}
}

View File

@@ -0,0 +1,65 @@
using System;
using Microsoft.Graph.Models;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
namespace Wino.Core.Extensions
{
public static class OutlookIntegratorExtensions
{
public static MailItemFolder GetLocalFolder(this MailFolder nativeFolder, Guid accountId)
{
return new MailItemFolder()
{
Id = Guid.NewGuid(),
FolderName = nativeFolder.DisplayName,
RemoteFolderId = nativeFolder.Id,
ParentRemoteFolderId = nativeFolder.ParentFolderId,
IsSynchronizationEnabled = true,
MailAccountId = accountId,
IsHidden = nativeFolder.IsHidden.GetValueOrDefault()
};
}
public static bool GetIsDraft(this Message message)
=> message != null && message.IsDraft.GetValueOrDefault();
public static bool GetIsRead(this Message message)
=> message != null && message.IsRead.GetValueOrDefault();
public static bool GetIsFocused(this Message message)
=> message?.InferenceClassification != null && message.InferenceClassification.Value == InferenceClassificationType.Focused;
public static bool GetIsFlagged(this Message message)
=> message?.Flag?.FlagStatus != null && message.Flag.FlagStatus == FollowupFlagStatus.Flagged;
public static MailCopy AsMailCopy(this Message outlookMessage)
{
bool isDraft = GetIsDraft(outlookMessage);
var mailCopy = new MailCopy()
{
MessageId = outlookMessage.InternetMessageId,
IsFlagged = GetIsFlagged(outlookMessage),
IsFocused = GetIsFocused(outlookMessage),
Importance = !outlookMessage.Importance.HasValue ? MailImportance.Normal : (MailImportance)outlookMessage.Importance.Value,
IsRead = GetIsRead(outlookMessage),
IsDraft = isDraft,
CreationDate = outlookMessage.ReceivedDateTime.GetValueOrDefault().DateTime,
HasAttachments = outlookMessage.HasAttachments.GetValueOrDefault(),
PreviewText = outlookMessage.BodyPreview,
Id = outlookMessage.Id,
ThreadId = outlookMessage.ConversationId,
FromName = outlookMessage.From?.EmailAddress?.Name,
FromAddress = outlookMessage.From?.EmailAddress?.Address,
Subject = outlookMessage.Subject,
FileId = Guid.NewGuid()
};
if (mailCopy.IsDraft)
mailCopy.DraftId = mailCopy.ThreadId;
return mailCopy;
}
}
}

View File

@@ -0,0 +1,22 @@
using System;
namespace Wino.Core.Extensions
{
public static class StringExtensions
{
public static bool Contains(this string source, string toCheck, StringComparison comp)
{
return source?.IndexOf(toCheck, comp) >= 0;
}
public static string ReplaceFirst(this string text, string search, string replace)
{
int pos = text.IndexOf(search);
if (pos < 0)
{
return text;
}
return text.Substring(0, pos) + replace + text.Substring(pos + search.Length);
}
}
}

View File

@@ -0,0 +1,10 @@
// This file is used by Code Analysis to maintain SuppressMessage
// attributes that are applied to this project.
// Project-level suppressions either have no target or are given
// a specific target and scoped to a namespace, type, member, etc.
using System.Diagnostics.CodeAnalysis;
[assembly: SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "<Pending>", Scope = "member", Target = "~P:Wino.Core.Models.IMailDisplayInformation.asd")]
[assembly: SuppressMessage("Minor Code Smell", "S3267:Loops should be simplified with \"LINQ\" expressions", Justification = "<Pending>", Scope = "member", Target = "~M:Wino.Core.Services.WinoRequestProcessor.PrepareRequestsAsync(Wino.Core.Domain.Enums.MailOperation,System.Collections.Generic.IEnumerable{System.String})~System.Threading.Tasks.Task{System.Collections.Generic.List{Wino.Core.Abstractions.Interfaces.Data.IWinoChangeRequest}}")]
[assembly: SuppressMessage("Minor Code Smell", "S3267:Loops should be simplified with \"LINQ\" expressions", Justification = "<Pending>", Scope = "member", Target = "~M:Wino.Core.Services.SynchronizationWorker.QueueAsync(System.Collections.Generic.IEnumerable{Wino.Core.Abstractions.Interfaces.Data.IWinoChangeRequest})")]

View File

@@ -0,0 +1,29 @@
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Google.Apis.Http;
using Wino.Domain.Entities;
namespace Wino.Core.Http
{
internal class GmailClientMessageHandler : ConfigurableMessageHandler
{
public Func<Task<TokenInformation>> TokenRetrieveDelegate { get; }
public GmailClientMessageHandler(Func<Task<TokenInformation>> tokenRetrieveDelegate) : base(new HttpClientHandler())
{
TokenRetrieveDelegate = tokenRetrieveDelegate;
}
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var tokenizationTask = TokenRetrieveDelegate.Invoke();
var tokenInformation = await tokenizationTask;
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", tokenInformation.AccessToken);
return await base.SendAsync(request, cancellationToken);
}
}
}

View File

@@ -0,0 +1,19 @@
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
namespace Wino.Core.Http
{
/// <summary>
/// Adds additional Prefer header for immutable id support in the Graph service client.
/// </summary>
public class MicrosoftImmutableIdHandler : DelegatingHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
request.Headers.TryAddWithoutValidation("Prefer", "IdType=\"ImmutableId\"");
return base.SendAsync(request, cancellationToken);
}
}
}

View File

@@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Kiota.Abstractions.Authentication;
using Wino.Domain.Entities;
using Wino.Domain.Interfaces;
namespace Wino.Core.Http
{
public class MicrosoftTokenProvider : IAccessTokenProvider
{
private readonly MailAccount _account;
private readonly IAuthenticator _authenticator;
public MicrosoftTokenProvider(MailAccount account, IAuthenticator authenticator)
{
_account = account;
_authenticator = authenticator;
}
public AllowedHostsValidator AllowedHostsValidator { get; }
public async Task<string> GetAuthorizationTokenAsync(Uri uri,
Dictionary<string, object> additionalAuthenticationContext = null,
CancellationToken cancellationToken = default)
{
var token = await _authenticator.GetTokenAsync(_account).ConfigureAwait(false);
return token?.AccessToken;
}
}
}

View File

@@ -0,0 +1,136 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using MailKit.Net.Imap;
using MoreLinq;
using Wino.Domain.Interfaces;
using Wino.Services.Requests.Bundles;
namespace Wino.Core.Integration
{
public abstract class BaseMailIntegrator<TNativeRequestType>
{
/// <summary>
/// How many items per single HTTP call can be modified.
/// </summary>
public abstract uint BatchModificationSize { get; }
/// <summary>
/// How many items must be downloaded per folder when the folder is first synchronized.
/// </summary>
public abstract uint InitialMessageDownloadCountPerFolder { get; }
/// <summary>
/// Creates a batched HttpBundle without a response for a collection of MailItem.
/// </summary>
/// <param name="batchChangeRequest">Generated batch request.</param>
/// <param name="action">An action to get the native request from the MailItem.</param>
/// <returns>Collection of http bundle that contains batch and native request.</returns>
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateBatchedHttpBundleFromGroup(
IBatchChangeRequest batchChangeRequest,
Func<IEnumerable<IRequest>, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
var groupedItems = batchChangeRequest.Items.Batch((int)BatchModificationSize);
foreach (var group in groupedItems)
yield return new HttpRequestBundle<TNativeRequestType>(action(group), batchChangeRequest);
}
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateBatchedHttpBundle(
IBatchChangeRequest batchChangeRequest,
Func<IRequest, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
var groupedItems = batchChangeRequest.Items.Batch((int)BatchModificationSize);
foreach (var group in groupedItems)
foreach (var item in group)
yield return new HttpRequestBundle<TNativeRequestType>(action(item), item);
yield break;
}
/// <summary>
/// Creates a single HttpBundle without a response for a collection of MailItem.
/// </summary>
/// <param name="batchChangeRequest">Batch request</param>
/// <param name="action">An action to get the native request from the MailItem</param>
/// <returns>Collection of http bundle that contains batch and native request.</returns>
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateHttpBundle(
IBatchChangeRequest batchChangeRequest,
Func<IRequest, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
foreach (var item in batchChangeRequest.Items)
yield return new HttpRequestBundle<TNativeRequestType>(action(item), batchChangeRequest);
}
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateHttpBundle<TResponseType>(
IBatchChangeRequest batchChangeRequest,
Func<IRequest, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
foreach (var item in batchChangeRequest.Items)
yield return new HttpRequestBundle<TNativeRequestType, TResponseType>(action(item), item);
}
/// <summary>
/// Creates HttpBundle with TResponse of expected response type from the http call for each of the items in the batch.
/// </summary>
/// <typeparam name="TResponse">Expected http response type after the call.</typeparam>
/// <param name="batchChangeRequest">Generated batch request.</param>
/// <param name="action">An action to get the native request from the MailItem.</param>
/// <returns>Collection of http bundle that contains batch and native request.</returns>
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateHttpBundleWithResponse<TResponse>(
IBatchChangeRequest batchChangeRequest,
Func<IRequest, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
foreach (var item in batchChangeRequest.Items)
yield return new HttpRequestBundle<TNativeRequestType, TResponse>(action(item), batchChangeRequest);
}
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateHttpBundleWithResponse<TResponse>(
IRequestBase item,
Func<IRequestBase, TNativeRequestType> action)
{
yield return new HttpRequestBundle<TNativeRequestType, TResponse>(action(item), item);
}
/// <summary>
/// Creates a batched HttpBundle with TResponse of expected response type from the http call for each of the items in the batch.
/// Func will be executed for each item separately in the batch request.
/// </summary>
/// <typeparam name="TResponse">Expected http response type after the call.</typeparam>
/// <param name="batchChangeRequest">Generated batch request.</param>
/// <param name="action">An action to get the native request from the MailItem.</param>
/// <returns>Collection of http bundle that contains batch and native request.</returns>
public IEnumerable<IRequestBundle<TNativeRequestType>> CreateBatchedHttpBundle<TResponse>(
IBatchChangeRequest batchChangeRequest,
Func<IRequest, TNativeRequestType> action)
{
if (batchChangeRequest.Items == null) yield break;
var groupedItems = batchChangeRequest.Items.Batch((int)BatchModificationSize);
foreach (var group in groupedItems)
foreach (var item in group)
yield return new HttpRequestBundle<TNativeRequestType, TResponse>(action(item), item);
yield break;
}
public IEnumerable<IRequestBundle<ImapRequest>> CreateTaskBundle(Func<ImapClient, Task> value, IRequestBase request)
{
var imapreq = new ImapRequest(value, request);
return [new ImapRequestBundle(imapreq, request)];
}
}
}

View File

@@ -0,0 +1,281 @@
using System;
using System.Collections.Concurrent;
using System.IO;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Net.Proxy;
using MailKit.Security;
using MoreLinq;
using Serilog;
using Wino.Domain.Exceptions;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
namespace Wino.Core.Integration
{
/// <summary>
/// Provides a pooling mechanism for ImapClient.
/// Makes sure that we don't have too many connections to the server.
/// Rents a connected & authenticated client from the pool all the time.
/// TODO: Keeps the clients alive by sending NOOP command periodically.
/// TODO: Listens to the Inbox folder for new messages.
/// </summary>
/// <param name="customServerInformation">Connection/Authentication info to be used to configure ImapClient.</param>
public class ImapClientPool : IDisposable
{
// Hardcoded implementation details for ID extension if the server supports.
// Some providers like Chinese 126 require Id to be sent before authentication.
// We don't expose any customer data here. Therefore it's safe for now.
// Later on maybe we can make it configurable and leave it to the user with passing
// real implementation details.
private readonly ImapImplementation _implementation = new ImapImplementation()
{
Version = "1.0",
OS = "Windows",
Vendor = "Wino"
};
private readonly int MinimumPoolSize = 5;
private readonly ConcurrentStack<ImapClient> _clients = [];
private readonly SemaphoreSlim _semaphore;
private readonly CustomServerInformation _customServerInformation;
private readonly Stream _protocolLogStream;
private readonly ILogger _logger = Log.ForContext<ImapClientPool>();
public ImapClientPool(CustomServerInformation customServerInformation, Stream protocolLogStream = null)
{
_customServerInformation = customServerInformation;
_protocolLogStream = protocolLogStream;
// Set the maximum pool size to 5 or the custom value if it's greater.
_semaphore = new(Math.Max(MinimumPoolSize, customServerInformation.MaxConcurrentClients));
}
private async Task EnsureConnectivityAsync(ImapClient client, bool isCreatedNew)
{
try
{
await EnsureConnectedAsync(client);
bool mustDoPostAuthIdentification = false;
if (isCreatedNew && client.IsConnected)
{
// Activate supported pre-auth capabilities.
if (client.Capabilities.HasFlag(ImapCapabilities.Compress))
await client.CompressAsync();
// Identify if the server supports ID extension.
// Some servers require it pre-authentication, some post-authentication.
// We'll observe the response here and do it after authentication if needed.
if (client.Capabilities.HasFlag(ImapCapabilities.Id))
{
try
{
await client.IdentifyAsync(_implementation);
}
catch (ImapCommandException commandException) when (commandException.Response == ImapCommandResponse.No || commandException.Response == ImapCommandResponse.Bad)
{
mustDoPostAuthIdentification = true;
}
catch (Exception)
{
throw;
}
}
}
await EnsureAuthenticatedAsync(client);
if (isCreatedNew && client.IsAuthenticated)
{
if (mustDoPostAuthIdentification) await client.IdentifyAsync(_implementation);
// Activate post-auth capabilities.
if (client.Capabilities.HasFlag(ImapCapabilities.QuickResync))
await client.EnableQuickResyncAsync();
}
}
catch (Exception ex)
{
throw new ImapClientPoolException(ex, GetProtocolLogContent());
}
finally
{
// Release it even if it fails.
_semaphore.Release();
}
}
public string GetProtocolLogContent()
{
if (_protocolLogStream == null) return default;
// Set the position to the beginning of the stream in case it is not already at the start
if (_protocolLogStream.CanSeek)
_protocolLogStream.Seek(0, SeekOrigin.Begin);
using var reader = new StreamReader(_protocolLogStream, Encoding.UTF8, true, 1024, leaveOpen: true);
return reader.ReadToEnd();
}
public async Task<ImapClient> GetClientAsync()
{
await _semaphore.WaitAsync();
if (_clients.TryPop(out ImapClient item))
{
await EnsureConnectivityAsync(item, false);
return item;
}
var client = CreateNewClient();
await EnsureConnectivityAsync(client, true);
return client;
}
public void Release(ImapClient item, bool destroyClient = false)
{
if (item != null)
{
if (destroyClient)
{
lock (item.SyncRoot)
{
item.Disconnect(true);
}
item.Dispose();
}
else
{
_clients.Push(item);
}
_semaphore.Release();
}
}
public void DestroyClient(ImapClient client)
{
if (client == null) return;
client.Disconnect(true);
client.Dispose();
}
private ImapClient CreateNewClient()
{
ImapClient client = null;
// Make sure to create a ImapClient with a protocol logger if enabled.
client = _protocolLogStream != null
? new ImapClient(new ProtocolLogger(_protocolLogStream))
: new ImapClient();
HttpProxyClient proxyClient = null;
// Add proxy client if exists.
if (!string.IsNullOrEmpty(_customServerInformation.ProxyServer))
{
proxyClient = new HttpProxyClient(_customServerInformation.ProxyServer, int.Parse(_customServerInformation.ProxyServerPort));
}
client.ProxyClient = proxyClient;
_logger.Debug("Created new ImapClient. Current clients: {Count}", _clients.Count);
return client;
}
private SecureSocketOptions GetSocketOptions(ImapConnectionSecurity connectionSecurity)
=> connectionSecurity switch
{
ImapConnectionSecurity.Auto => SecureSocketOptions.Auto,
ImapConnectionSecurity.None => SecureSocketOptions.None,
ImapConnectionSecurity.StartTls => SecureSocketOptions.StartTlsWhenAvailable,
ImapConnectionSecurity.SslTls => SecureSocketOptions.SslOnConnect,
_ => SecureSocketOptions.None
};
public async Task EnsureConnectedAsync(ImapClient client)
{
if (client.IsConnected) return;
await client.ConnectAsync(_customServerInformation.IncomingServer,
int.Parse(_customServerInformation.IncomingServerPort),
GetSocketOptions(_customServerInformation.IncomingServerSocketOption));
}
public async Task EnsureAuthenticatedAsync(ImapClient client)
{
if (client.IsAuthenticated) return;
var cred = new NetworkCredential(_customServerInformation.IncomingServerUsername, _customServerInformation.IncomingServerPassword);
var prefferedAuthenticationMethod = _customServerInformation.IncomingAuthenticationMethod;
if (prefferedAuthenticationMethod != ImapAuthenticationMethod.Auto)
{
// Anything beside Auto must be explicitly set for the client.
client.AuthenticationMechanisms.Clear();
var saslMechanism = GetSASLAuthenticationMethodName(prefferedAuthenticationMethod);
client.AuthenticationMechanisms.Add(saslMechanism);
await client.AuthenticateAsync(SaslMechanism.Create(saslMechanism, cred));
}
else
{
await client.AuthenticateAsync(cred);
}
}
private string GetSASLAuthenticationMethodName(ImapAuthenticationMethod method)
{
return method switch
{
ImapAuthenticationMethod.NormalPassword => "PLAIN",
ImapAuthenticationMethod.EncryptedPassword => "LOGIN",
ImapAuthenticationMethod.Ntlm => "NTLM",
ImapAuthenticationMethod.CramMd5 => "CRAM-MD5",
ImapAuthenticationMethod.DigestMd5 => "DIGEST-MD5",
_ => "PLAIN"
};
}
public void Dispose()
{
_clients.ForEach(client =>
{
lock (client.SyncRoot)
{
client.Disconnect(true);
}
});
_clients.ForEach(client =>
{
client.Dispose();
});
_clients.Clear();
if (_protocolLogStream != null)
{
_protocolLogStream.Dispose();
}
}
}
}

View File

@@ -0,0 +1,9 @@
using MailKit;
namespace Wino.Core.Mime
{
/// <summary>
/// Encapsulates all required information to create a MimeMessage for IMAP synchronizer.
/// </summary>
public record ImapMessageCreationPackage(IMessageSummary MessageSummary, IMailFolder MailFolder);
}

View File

@@ -0,0 +1,36 @@
using System.Collections.Generic;
using Wino.Domain.Interfaces;
using Wino.Services.Requests;
namespace Wino.Core.Misc
{
/// <summary>
/// This is incomplete.
/// </summary>
internal class RequestComparer : IEqualityComparer<IRequestBase>
{
public bool Equals(IRequestBase x, IRequestBase y)
{
if (x is MoveRequest sourceMoveRequest && y is MoveRequest targetMoveRequest)
{
return sourceMoveRequest.FromFolder.Id == targetMoveRequest.FromFolder.Id && sourceMoveRequest.ToFolder.Id == targetMoveRequest.ToFolder.Id;
}
else if (x is ChangeFlagRequest sourceFlagRequest && y is ChangeFlagRequest targetFlagRequest)
{
return sourceFlagRequest.IsFlagged == targetFlagRequest.IsFlagged;
}
else if (x is MarkReadRequest sourceMarkReadRequest && y is MarkReadRequest targetMarkReadRequest)
{
return sourceMarkReadRequest.Item.IsRead == targetMarkReadRequest.Item.IsRead;
}
else if (x is DeleteRequest sourceDeleteRequest && y is DeleteRequest targetDeleteRequest)
{
return sourceDeleteRequest.MailItem.AssignedFolder.Id == targetDeleteRequest.MailItem.AssignedFolder.Id;
}
return true;
}
public int GetHashCode(IRequestBase obj) => obj.Operation.GetHashCode();
}
}

View File

@@ -0,0 +1,51 @@
using System.IO;
using System.Threading.Tasks;
using Wino.Core.Integration;
using Wino.Domain;
using Wino.Domain.Entities;
using Wino.Domain.Interfaces;
namespace Wino.Core.Services
{
public class ImapTestService : IImapTestService
{
private readonly IPreferencesService _preferencesService;
private readonly IApplicationConfiguration _appInitializerService;
private Stream _protocolLogStream;
public ImapTestService(IPreferencesService preferencesService, IApplicationConfiguration appInitializerService)
{
_preferencesService = preferencesService;
_appInitializerService = appInitializerService;
}
private void EnsureProtocolLogFileExists()
{
// Create new file for protocol logger.
var localAppFolderPath = _appInitializerService.ApplicationDataFolderPath;
var logFile = Path.Combine(localAppFolderPath, Constants.ProtocolLogFileName);
if (File.Exists(logFile))
File.Delete(logFile);
_protocolLogStream = File.Create(logFile);
}
public async Task TestImapConnectionAsync(CustomServerInformation serverInformation)
{
EnsureProtocolLogFileExists();
var clientPool = new ImapClientPool(serverInformation, _protocolLogStream);
using (clientPool)
{
// This call will make sure that everything is authenticated + connected successfully.
var client = await clientPool.GetClientAsync();
clientPool.Release(client);
}
}
}
}

View File

@@ -0,0 +1,14 @@
using Microsoft.Extensions.DependencyInjection;
using Wino.Core.Services;
using Wino.Domain.Interfaces;
namespace Wino.Synchronization
{
public static class SynchronizationContainerSetup
{
public static void RegisterSynchronizationServices(this IServiceCollection services)
{
services.AddTransient<IImapTestService, ImapTestService>();
}
}
}

View File

@@ -0,0 +1,313 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using CommunityToolkit.Mvvm.Messaging;
using MailKit;
using Serilog;
using Wino.Core.Integration;
using Wino.Core.Misc;
using Wino.Domain;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Wino.Domain.Interfaces;
using Wino.Domain.Models.MailItem;
using Wino.Domain.Models.Synchronization;
using Wino.Messaging.Client.Mails;
using Wino.Messaging.Client.Synchronization;
using Wino.Services.Requests;
namespace Wino.Core.Synchronizers
{
public abstract class BaseSynchronizer<TBaseRequest, TMessageType> : BaseMailIntegrator<TBaseRequest>, IBaseSynchronizer
{
private SemaphoreSlim synchronizationSemaphore = new(1);
private CancellationToken activeSynchronizationCancellationToken;
protected ConcurrentBag<IRequestBase> changeRequestQueue = [];
protected ILogger Logger = Log.ForContext<BaseSynchronizer<TBaseRequest, TMessageType>>();
protected BaseSynchronizer(MailAccount account)
{
Account = account;
}
public MailAccount Account { get; }
private AccountSynchronizerState state;
public AccountSynchronizerState State
{
get { return state; }
private set
{
state = value;
WeakReferenceMessenger.Default.Send(new AccountSynchronizerStateChanged(this, value));
}
}
/// <summary>
/// Queues a single request to be executed in the next synchronization.
/// </summary>
/// <param name="request">Request to execute.</param>
public void QueueRequest(IRequestBase request) => changeRequestQueue.Add(request);
/// <summary>
/// Creates a new Wino Mail Item package out of native message type with full Mime.
/// </summary>
/// <param name="message">Native message type for the synchronizer.</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Package that encapsulates downloaded Mime and additional information for adding new mail.</returns>
public abstract Task<List<NewMailItemPackage>> CreateNewMailPackagesAsync(TMessageType message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default);
/// <summary>
/// Runs existing queued requests in the queue.
/// </summary>
/// <param name="batchedRequests">Batched requests to execute. Integrator methods will only receive batched requests.</param>
/// <param name="cancellationToken">Cancellation token</param>
public abstract Task ExecuteNativeRequestsAsync(IEnumerable<IRequestBundle<TBaseRequest>> batchedRequests, CancellationToken cancellationToken = default);
public abstract Task<SynchronizationResult> SynchronizeInternalAsync(SynchronizationOptions options, CancellationToken cancellationToken = default);
public async Task<SynchronizationResult> SynchronizeAsync(SynchronizationOptions options, CancellationToken cancellationToken = default)
{
try
{
activeSynchronizationCancellationToken = cancellationToken;
var batches = CreateBatchRequests().Distinct();
if (batches.Any())
{
Logger.Information($"{batches?.Count() ?? 0} batched requests");
State = AccountSynchronizerState.ExecutingRequests;
var nativeRequests = CreateNativeRequestBundles(batches);
Console.WriteLine($"Prepared {nativeRequests.Count()} native requests");
await ExecuteNativeRequestsAsync(nativeRequests, activeSynchronizationCancellationToken);
PublishUnreadItemChanges();
// Execute request sync options should be re-calculated after execution.
// This is the part we decide which individual folders must be synchronized
// after the batch request execution.
if (options.Type == SynchronizationType.ExecuteRequests)
options = GetSynchronizationOptionsAfterRequestExecution(batches);
}
State = AccountSynchronizerState.Synchronizing;
await synchronizationSemaphore.WaitAsync(activeSynchronizationCancellationToken);
// Let servers to finish their job. Sometimes the servers doesn't respond immediately.
bool shouldDelayExecution = batches.Any(a => a.DelayExecution);
if (shouldDelayExecution)
{
await Task.Delay(2000);
}
// Start the internal synchronization.
var synchronizationResult = await SynchronizeInternalAsync(options, activeSynchronizationCancellationToken).ConfigureAwait(false);
PublishUnreadItemChanges();
return synchronizationResult;
}
catch (OperationCanceledException)
{
Logger.Warning("Synchronization cancelled.");
return SynchronizationResult.Canceled;
}
catch (Exception)
{
// Disable maybe?
throw;
}
finally
{
// Reset account progress to hide the progress.
options.ProgressListener?.AccountProgressUpdated(Account.Id, 0);
State = AccountSynchronizerState.Idle;
synchronizationSemaphore.Release();
}
}
/// <summary>
/// Updates unread item counts for some folders and account.
/// Sends a message that shell can pick up and update the UI.
/// </summary>
private void PublishUnreadItemChanges()
=> WeakReferenceMessenger.Default.Send(new RefreshUnreadCountsMessage(Account.Id));
/// <summary>
/// 1. Group all requests by operation type.
/// 2. Group all individual operation type requests with equality check.
/// Equality comparison in the records are done with RequestComparer
/// to ignore Item property. Each request can have their own logic for comparison.
/// For example, move requests for different mails from the same folder to the same folder
/// must be dispatched in the same batch. This is much faster for the server. Specially IMAP
/// since all folders must be asynchronously opened/closed.
/// </summary>
/// <returns>Batch request collection for all these single requests.</returns>
private List<IRequestBase> CreateBatchRequests()
{
var batchList = new List<IRequestBase>();
var comparer = new RequestComparer();
while (changeRequestQueue.Count > 0)
{
if (changeRequestQueue.TryPeek(out IRequestBase request))
{
// Mail request, must be batched.
if (request is IRequest mailRequest)
{
var equalItems = changeRequestQueue
.Where(a => a is IRequest && comparer.Equals(a, request))
.Cast<IRequest>()
.ToList();
batchList.Add(mailRequest.CreateBatch(equalItems));
// Remove these items from the queue.
foreach (var item in equalItems)
{
changeRequestQueue.TryTake(out _);
}
}
else if (changeRequestQueue.TryTake(out request))
{
// This is a folder operation.
// There is no need to batch them since Users can't do folder ops in bulk.
batchList.Add(request);
}
}
}
return batchList;
}
/// <summary>
/// Converts batched requests into HTTP/Task calls that derived synchronizers can execute.
/// </summary>
/// <param name="batchChangeRequests">Batch requests to be converted.</param>
/// <returns>Collection of native requests for individual synchronizer type.</returns>
private IEnumerable<IRequestBundle<TBaseRequest>> CreateNativeRequestBundles(IEnumerable<IRequestBase> batchChangeRequests)
{
IEnumerable<IEnumerable<IRequestBundle<TBaseRequest>>> GetNativeRequests()
{
foreach (var item in batchChangeRequests)
{
switch (item.Operation)
{
case MailSynchronizerOperation.Send:
yield return SendDraft((BatchSendDraftRequestRequest)item);
break;
case MailSynchronizerOperation.MarkRead:
yield return MarkRead((BatchMarkReadRequest)item);
break;
case MailSynchronizerOperation.Move:
yield return Move((BatchMoveRequest)item);
break;
case MailSynchronizerOperation.Delete:
yield return Delete((BatchDeleteRequest)item);
break;
case MailSynchronizerOperation.ChangeFlag:
yield return ChangeFlag((BatchChangeFlagRequest)item);
break;
case MailSynchronizerOperation.AlwaysMoveTo:
yield return AlwaysMoveTo((BatchAlwaysMoveToRequest)item);
break;
case MailSynchronizerOperation.MoveToFocused:
yield return MoveToFocused((BatchMoveToFocusedRequest)item);
break;
case MailSynchronizerOperation.CreateDraft:
yield return CreateDraft((BatchCreateDraftRequest)item);
break;
case MailSynchronizerOperation.RenameFolder:
yield return RenameFolder((RenameFolderRequest)item);
break;
case MailSynchronizerOperation.EmptyFolder:
yield return EmptyFolder((EmptyFolderRequest)item);
break;
case MailSynchronizerOperation.MarkFolderRead:
yield return MarkFolderAsRead((MarkFolderAsReadRequest)item);
break;
case MailSynchronizerOperation.Archive:
yield return Archive((BatchArchiveRequest)item);
break;
}
}
};
return GetNativeRequests().SelectMany(collections => collections);
}
/// <summary>
/// Attempts to find out the best possible synchronization options after the batch request execution.
/// </summary>
/// <param name="batches">Batch requests to run in synchronization.</param>
/// <returns>New synchronization options with minimal HTTP effort.</returns>
private SynchronizationOptions GetSynchronizationOptionsAfterRequestExecution(IEnumerable<IRequestBase> requests)
{
bool isAllCustomSynchronizationRequests = requests.All(a => a is ICustomFolderSynchronizationRequest);
var options = new SynchronizationOptions()
{
AccountId = Account.Id,
Type = SynchronizationType.FoldersOnly
};
if (isAllCustomSynchronizationRequests)
{
// Gather FolderIds to synchronize.
options.Type = SynchronizationType.Custom;
options.SynchronizationFolderIds = requests.Cast<ICustomFolderSynchronizationRequest>().SelectMany(a => a.SynchronizationFolderIds).ToList();
}
else
{
// At this point it's a mix of everything. Do full sync.
options.Type = SynchronizationType.Full;
}
return options;
}
public virtual bool DelaySendOperationSynchronization() => false;
public virtual IEnumerable<IRequestBundle<TBaseRequest>> Move(BatchMoveRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> ChangeFlag(BatchChangeFlagRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> MarkRead(BatchMarkReadRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> Delete(BatchDeleteRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> AlwaysMoveTo(BatchAlwaysMoveToRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> MoveToFocused(BatchMoveToFocusedRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> CreateDraft(BatchCreateDraftRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> SendDraft(BatchSendDraftRequestRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> RenameFolder(RenameFolderRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> EmptyFolder(EmptyFolderRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> MarkFolderAsRead(MarkFolderAsReadRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public virtual IEnumerable<IRequestBundle<TBaseRequest>> Archive(BatchArchiveRequest request) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
/// <summary>
/// Downloads a single missing message from synchronizer and saves it to given FileId from IMailItem.
/// </summary>
/// <param name="mailItem">Mail item that its mime file does not exist on the disk.</param>
/// <param name="transferProgress">Optional download progress for IMAP synchronizer.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public virtual Task DownloadMissingMimeMessageAsync(IMailItem mailItem, ITransferProgress transferProgress = null, CancellationToken cancellationToken = default) => throw new NotSupportedException(string.Format(Translator.Exception_UnsupportedSynchronizerOperation, this.GetType()));
public bool CancelActiveSynchronization()
{
// TODO: What if account is deleted during synchronization?
return true;
}
}
}

View File

@@ -0,0 +1,970 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Google.Apis.Gmail.v1;
using Google.Apis.Gmail.v1.Data;
using Google.Apis.Http;
using Google.Apis.Requests;
using Google.Apis.Services;
using MailKit;
using Microsoft.IdentityModel.Tokens;
using MimeKit;
using MoreLinq;
using Serilog;
using Wino.Core.Extensions;
using Wino.Core.Http;
using Wino.Domain;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Wino.Domain.Exceptions;
using Wino.Domain.Interfaces;
using Wino.Domain.Models.MailItem;
using Wino.Domain.Models.Synchronization;
using Wino.Services.Requests;
using Wino.Services.Requests.Bundles;
namespace Wino.Core.Synchronizers
{
public class GmailSynchronizer : BaseSynchronizer<IClientServiceRequest, Message>, IHttpClientFactory
{
public override uint BatchModificationSize => 1000;
public override uint InitialMessageDownloadCountPerFolder => 1200;
// It's actually 100. But Gmail SDK has internal bug for Out of Memory exception.
// https://github.com/googleapis/google-api-dotnet-client/issues/2603
private const uint MaximumAllowedBatchRequestSize = 10;
private readonly ConfigurableHttpClient _gmailHttpClient;
private readonly GmailService _gmailService;
private readonly IAuthenticator _authenticator;
private readonly IGmailChangeProcessor _gmailChangeProcessor;
private readonly ILogger _logger = Log.ForContext<GmailSynchronizer>();
public GmailSynchronizer(MailAccount account,
IAuthenticator authenticator,
IGmailChangeProcessor gmailChangeProcessor) : base(account)
{
var messageHandler = new GmailClientMessageHandler(() => _authenticator.GetTokenAsync(Account));
var initializer = new BaseClientService.Initializer()
{
HttpClientFactory = this
};
_gmailHttpClient = new ConfigurableHttpClient(messageHandler);
_gmailService = new GmailService(initializer);
_authenticator = authenticator;
_gmailChangeProcessor = gmailChangeProcessor;
}
public ConfigurableHttpClient CreateHttpClient(CreateHttpClientArgs args) => _gmailHttpClient;
public override async Task<SynchronizationResult> SynchronizeInternalAsync(SynchronizationOptions options, CancellationToken cancellationToken = default)
{
_logger.Information("Internal synchronization started for {Name}", Account.Name);
// Gmail must always synchronize folders before because it doesn't have a per-folder sync.
bool shouldSynchronizeFolders = true;
if (shouldSynchronizeFolders)
{
_logger.Information("Synchronizing folders for {Name}", Account.Name);
await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false);
_logger.Information("Synchronizing folders for {Name} is completed", Account.Name);
}
// There is no specific folder synchronization in Gmail.
// Therefore we need to stop the synchronization at this point
// if type is only folder metadata sync.
if (options.Type == SynchronizationType.FoldersOnly) return SynchronizationResult.Empty;
cancellationToken.ThrowIfCancellationRequested();
bool isInitialSync = string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier);
_logger.Debug("Is initial synchronization: {IsInitialSync}", isInitialSync);
var missingMessageIds = new List<string>();
var deltaChanges = new List<ListHistoryResponse>(); // For tracking delta changes.
var listChanges = new List<ListMessagesResponse>(); // For tracking initial sync changes.
/* Processing flow order is important to preserve the validity of history.
* 1 - Process added mails. Because we need to create the mail first before assigning it to labels.
* 2 - Process label assignments.
* 3 - Process removed mails.
* This affects reporting progres if done individually for each history change.
* Therefore we need to process all changes in one go after the fetch.
*/
if (isInitialSync)
{
// Initial synchronization.
// Google sends message id and thread id in this query.
// We'll collect them and send a Batch request to get details of the messages.
var messageRequest = _gmailService.Users.Messages.List("me");
// Gmail doesn't do per-folder sync. So our per-folder count is the same as total message count.
messageRequest.MaxResults = InitialMessageDownloadCountPerFolder;
messageRequest.IncludeSpamTrash = true;
ListMessagesResponse result = null;
string nextPageToken = string.Empty;
while (true)
{
if (!string.IsNullOrEmpty(nextPageToken))
{
messageRequest.PageToken = nextPageToken;
}
result = await messageRequest.ExecuteAsync(cancellationToken);
nextPageToken = result.NextPageToken;
listChanges.Add(result);
// Nothing to fetch anymore. Break the loop.
if (nextPageToken == null)
break;
}
}
else
{
var startHistoryId = ulong.Parse(Account.SynchronizationDeltaIdentifier);
var nextPageToken = ulong.Parse(Account.SynchronizationDeltaIdentifier).ToString();
var historyRequest = _gmailService.Users.History.List("me");
historyRequest.StartHistoryId = startHistoryId;
while (!string.IsNullOrEmpty(nextPageToken))
{
// If this is the first delta check, start from the last history id.
// Otherwise start from the next page token. We set them both to the same value for start.
// For each different page we set the page token to the next page token.
bool isFirstDeltaCheck = nextPageToken == startHistoryId.ToString();
if (!isFirstDeltaCheck)
historyRequest.PageToken = nextPageToken;
var historyResponse = await historyRequest.ExecuteAsync(cancellationToken);
nextPageToken = historyResponse.NextPageToken;
if (historyResponse.History == null)
continue;
deltaChanges.Add(historyResponse);
}
}
// Add initial message ids from initial sync.
missingMessageIds.AddRange(listChanges.Where(a => a.Messages != null).SelectMany(a => a.Messages).Select(a => a.Id));
// Add missing message ids from delta changes.
foreach (var historyResponse in deltaChanges)
{
var addedMessageIds = historyResponse.History
.Where(a => a.MessagesAdded != null)
.SelectMany(a => a.MessagesAdded)
.Where(a => a.Message != null)
.Select(a => a.Message.Id);
missingMessageIds.AddRange(addedMessageIds);
}
// Consolidate added/deleted elements.
// For example: History change might report downloading a mail first, then deleting it in another history change.
// In that case, downloading mail will return entity not found error.
// Plus, it's a redundant download the mail.
// Purge missing message ids from potentially deleted mails to prevent this.
var messageDeletedHistoryChanges = deltaChanges
.Where(a => a.History != null)
.SelectMany(a => a.History)
.Where(a => a.MessagesDeleted != null)
.SelectMany(a => a.MessagesDeleted);
var deletedMailIdsInHistory = messageDeletedHistoryChanges.Select(a => a.Message.Id);
if (deletedMailIdsInHistory.Any())
{
var mailIdsToConsolidate = missingMessageIds.Where(a => deletedMailIdsInHistory.Contains(a)).ToList();
int consolidatedMessageCount = missingMessageIds.RemoveAll(a => deletedMailIdsInHistory.Contains(a));
if (consolidatedMessageCount > 0)
{
// TODO: Also delete the history changes that are related to these mails.
// This will prevent unwanted logs and additional queries to look for them in processing.
_logger.Information($"Purged {consolidatedMessageCount} missing mail downloads. ({string.Join(",", mailIdsToConsolidate)})");
}
}
// Start downloading missing messages.
await BatchDownloadMessagesAsync(missingMessageIds, options.ProgressListener, cancellationToken).ConfigureAwait(false);
// Map remote drafts to local drafts.
await MapDraftIdsAsync(cancellationToken).ConfigureAwait(false);
// Start processing delta changes.
foreach (var historyResponse in deltaChanges)
{
await ProcessHistoryChangesAsync(historyResponse).ConfigureAwait(false);
}
// Take the max history id from delta changes and update the account sync modifier.
var maxHistoryId = deltaChanges.Max(a => a.HistoryId);
if (maxHistoryId != null)
{
// TODO: This is not good. Centralize the identifier fetch and prevent direct access here.
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, maxHistoryId.ToString()).ConfigureAwait(false);
_logger.Debug("Final sync identifier {SynchronizationDeltaIdentifier}", Account.SynchronizationDeltaIdentifier);
}
// Get all unred new downloaded items and return in the result.
// This is primarily used in notifications.
var unreadNewItems = await _gmailChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, missingMessageIds).ConfigureAwait(false);
return SynchronizationResult.Completed(unreadNewItems);
}
private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default)
{
try
{
var localFolders = await _gmailChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
var folderRequest = _gmailService.Users.Labels.List("me");
var labelsResponse = await folderRequest.ExecuteAsync(cancellationToken).ConfigureAwait(false);
if (labelsResponse.Labels == null)
{
_logger.Warning("No folders found for {Name}", Account.Name);
return;
}
List<MailItemFolder> insertedFolders = new();
List<MailItemFolder> updatedFolders = new();
List<MailItemFolder> deletedFolders = new();
// 1. Handle deleted labels.
foreach (var localFolder in localFolders)
{
// Category folder is virtual folder for Wino. Skip it.
if (localFolder.SpecialFolderType == SpecialFolderType.Category) continue;
var remoteFolder = labelsResponse.Labels.FirstOrDefault(a => a.Id == localFolder.RemoteFolderId);
if (remoteFolder == null)
{
// Local folder doesn't exists remotely. Delete local copy.
await _gmailChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false);
deletedFolders.Add(localFolder);
}
}
// Delete the deleted folders from local list.
deletedFolders.ForEach(a => localFolders.Remove(a));
// 2. Handle update/insert based on remote folders.
foreach (var remoteFolder in labelsResponse.Labels)
{
var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.Id);
if (existingLocalFolder == null)
{
// Insert new folder.
var localFolder = remoteFolder.GetLocalFolder(labelsResponse, Account.Id);
insertedFolders.Add(localFolder);
}
else
{
// Update existing folder. Right now we only update the name.
// TODO: Moving folders around different parents. This is not supported right now.
// We will need more comphrensive folder update mechanism to support this.
if (ShouldUpdateFolder(remoteFolder, existingLocalFolder))
{
existingLocalFolder.FolderName = remoteFolder.Name;
updatedFolders.Add(existingLocalFolder);
}
else
{
// Remove it from the local folder list to skip additional folder updates.
localFolders.Remove(existingLocalFolder);
}
}
}
// 3.Process changes in order-> Insert, Update. Deleted ones are already processed.
foreach (var folder in insertedFolders)
{
await _gmailChangeProcessor.InsertFolderAsync(folder).ConfigureAwait(false);
}
foreach (var folder in updatedFolders)
{
await _gmailChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false);
}
}
catch (Exception)
{
throw;
}
}
private bool ShouldUpdateFolder(Label remoteFolder, MailItemFolder existingLocalFolder)
=> existingLocalFolder.FolderName.Equals(GoogleIntegratorExtensions.GetFolderName(remoteFolder), StringComparison.OrdinalIgnoreCase) == false;
/// <summary>
/// Returns a single get request to retrieve the raw message with the given id
/// </summary>
/// <param name="messageId">Message to download.</param>
/// <returns>Get request for raw mail.</returns>
private UsersResource.MessagesResource.GetRequest CreateSingleMessageGet(string messageId)
{
var singleRequest = _gmailService.Users.Messages.Get("me", messageId);
singleRequest.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Raw;
return singleRequest;
}
/// <summary>
/// Downloads given message ids per batch and processes them.
/// </summary>
/// <param name="messageIds">Gmail message ids to download.</param>
/// <param name="cancellationToken">Cancellation token.</param>
private async Task BatchDownloadMessagesAsync(IEnumerable<string> messageIds, ISynchronizationProgress progressListener = null, CancellationToken cancellationToken = default)
{
var totalDownloadCount = messageIds.Count();
if (totalDownloadCount == 0) return;
var downloadedItemCount = 0;
_logger.Debug("Batch downloading {Count} messages for {Name}", messageIds.Count(), Account.Name);
var allDownloadRequests = messageIds.Select(CreateSingleMessageGet);
// Respect the batch size limit for batch requests.
var batchedDownloadRequests = allDownloadRequests.Batch((int)MaximumAllowedBatchRequestSize);
_logger.Debug("Total items to download: {TotalDownloadCount}. Created {Count} batch download requests for {Name}.", batchedDownloadRequests.Count(), Account.Name, totalDownloadCount);
// Gmail SDK's BatchRequest has Action delegate for callback, not Task.
// Therefore it's not possible to make sure that downloaded item is processed in the database before this
// async callback is finished. Therefore we need to wrap all local database processings into task list and wait all of them to finish
// Batch execution finishes after response parsing is done.
var batchProcessCallbacks = new List<Task>();
foreach (var batchBundle in batchedDownloadRequests)
{
cancellationToken.ThrowIfCancellationRequested();
var batchRequest = new BatchRequest(_gmailService);
// Queue each request into this batch.
batchBundle.ForEach(request =>
{
batchRequest.Queue<Message>(request, (content, error, index, message) =>
{
var downloadingMessageId = messageIds.ElementAt(index);
batchProcessCallbacks.Add(HandleSingleItemDownloadedCallbackAsync(content, error, downloadingMessageId, cancellationToken));
downloadedItemCount++;
var progressValue = downloadedItemCount * 100 / Math.Max(1, totalDownloadCount);
progressListener?.AccountProgressUpdated(Account.Id, progressValue);
});
});
_logger.Information("Executing batch download with {Count} items.", batchRequest.Count);
await batchRequest.ExecuteAsync(cancellationToken);
// This is important due to bug in Gmail SDK.
// We force GC here to prevent Out of Memory exception.
// https://github.com/googleapis/google-api-dotnet-client/issues/2603
GC.Collect();
}
// Wait for all processing to finish.
await Task.WhenAll(batchProcessCallbacks).ConfigureAwait(false);
}
/// <summary>
/// Processes the delta changes for the given history changes.
/// Message downloads are not handled here since it's better to batch them.
/// </summary>
/// <param name="listHistoryResponse">List of history changes.</param>
private async Task ProcessHistoryChangesAsync(ListHistoryResponse listHistoryResponse)
{
_logger.Debug("Processing delta change {HistoryId} for {Name}", Account.Name, listHistoryResponse.HistoryId.GetValueOrDefault());
foreach (var history in listHistoryResponse.History)
{
// Handle label additions.
if (history.LabelsAdded is not null)
{
foreach (var addedLabel in history.LabelsAdded)
{
await HandleLabelAssignmentAsync(addedLabel);
}
}
// Handle label removals.
if (history.LabelsRemoved is not null)
{
foreach (var removedLabel in history.LabelsRemoved)
{
await HandleLabelRemovalAsync(removedLabel);
}
}
// Handle removed messages.
if (history.MessagesDeleted is not null)
{
foreach (var deletedMessage in history.MessagesDeleted)
{
var messageId = deletedMessage.Message.Id;
_logger.Debug("Processing message deletion for {MessageId}", messageId);
await _gmailChangeProcessor.DeleteMailAsync(Account.Id, messageId).ConfigureAwait(false);
}
}
}
}
private async Task HandleLabelAssignmentAsync(HistoryLabelAdded addedLabel)
{
var messageId = addedLabel.Message.Id;
_logger.Debug("Processing label assignment for message {MessageId}", messageId);
foreach (var labelId in addedLabel.LabelIds)
{
// When UNREAD label is added mark the message as un-read.
if (labelId == GoogleIntegratorExtensions.UNREAD_LABEL_ID)
await _gmailChangeProcessor.ChangeMailReadStatusAsync(messageId, false).ConfigureAwait(false);
// When STARRED label is added mark the message as flagged.
if (labelId == GoogleIntegratorExtensions.STARRED_LABEL_ID)
await _gmailChangeProcessor.ChangeFlagStatusAsync(messageId, true).ConfigureAwait(false);
await _gmailChangeProcessor.CreateAssignmentAsync(Account.Id, messageId, labelId).ConfigureAwait(false);
}
}
private async Task HandleLabelRemovalAsync(HistoryLabelRemoved removedLabel)
{
var messageId = removedLabel.Message.Id;
_logger.Debug("Processing label removed for message {MessageId}", messageId);
foreach (var labelId in removedLabel.LabelIds)
{
// When UNREAD label is removed mark the message as read.
if (labelId == GoogleIntegratorExtensions.UNREAD_LABEL_ID)
await _gmailChangeProcessor.ChangeMailReadStatusAsync(messageId, true).ConfigureAwait(false);
// When STARRED label is removed mark the message as un-flagged.
if (labelId == GoogleIntegratorExtensions.STARRED_LABEL_ID)
await _gmailChangeProcessor.ChangeFlagStatusAsync(messageId, false).ConfigureAwait(false);
// For other labels remove the mail assignment.
await _gmailChangeProcessor.DeleteAssignmentAsync(Account.Id, messageId, labelId).ConfigureAwait(false);
}
}
/// <summary>
/// Prepares Gmail Draft object from Google SDK.
/// If provided, ThreadId ties the draft to a thread. Used when replying messages.
/// If provided, DraftId updates the draft instead of creating a new one.
/// </summary>
/// <param name="mimeMessage">MailKit MimeMessage to include as raw message into Gmail request.</param>
/// <param name="messageThreadId">ThreadId that this draft should be tied to.</param>
/// <param name="messageDraftId">Existing DraftId from Gmail to update existing draft.</param>
/// <returns></returns>
private Draft PrepareGmailDraft(MimeMessage mimeMessage, string messageThreadId = "", string messageDraftId = "")
{
mimeMessage.Prepare(EncodingConstraint.None);
var mimeString = mimeMessage.ToString();
var base64UrlEncodedMime = Base64UrlEncoder.Encode(mimeString);
var nativeMessage = new Message()
{
Raw = base64UrlEncodedMime,
};
if (!string.IsNullOrEmpty(messageThreadId))
nativeMessage.ThreadId = messageThreadId;
var draft = new Draft()
{
Message = nativeMessage,
Id = messageDraftId
};
return draft;
}
#region Mail Integrations
public override IEnumerable<IRequestBundle<IClientServiceRequest>> Move(BatchMoveRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (items) =>
{
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = items.Select(a => a.Item.Id.ToString()).ToList(),
AddLabelIds = new[] { request.ToFolder.RemoteFolderId },
RemoveLabelIds = new[] { request.FromFolder.RemoteFolderId }
};
return _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> ChangeFlag(BatchChangeFlagRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (items) =>
{
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = items.Select(a => a.Item.Id.ToString()).ToList(),
};
if (request.IsFlagged)
batchModifyRequest.AddLabelIds = new List<string>() { GoogleIntegratorExtensions.STARRED_LABEL_ID };
else
batchModifyRequest.RemoveLabelIds = new List<string>() { GoogleIntegratorExtensions.STARRED_LABEL_ID };
return _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> MarkRead(BatchMarkReadRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (items) =>
{
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = items.Select(a => a.Item.Id.ToString()).ToList(),
};
if (request.IsRead)
batchModifyRequest.RemoveLabelIds = new List<string>() { GoogleIntegratorExtensions.UNREAD_LABEL_ID };
else
batchModifyRequest.AddLabelIds = new List<string>() { GoogleIntegratorExtensions.UNREAD_LABEL_ID };
return _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> Delete(BatchDeleteRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (items) =>
{
var batchModifyRequest = new BatchDeleteMessagesRequest
{
Ids = items.Select(a => a.Item.Id.ToString()).ToList(),
};
return _gmailService.Users.Messages.BatchDelete(batchModifyRequest, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> CreateDraft(BatchCreateDraftRequest request)
{
return CreateHttpBundle(request, (item) =>
{
if (item is not CreateDraftRequest singleRequest)
throw new ArgumentException("BatchCreateDraftRequest collection must be of type CreateDraftRequest.");
Draft draft = null;
// It's new mail. Not a reply
if (singleRequest.DraftPreperationRequest.ReferenceMailCopy == null)
draft = PrepareGmailDraft(singleRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage);
else
draft = PrepareGmailDraft(singleRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage,
singleRequest.DraftPreperationRequest.ReferenceMailCopy.ThreadId,
singleRequest.DraftPreperationRequest.ReferenceMailCopy.DraftId);
return _gmailService.Users.Drafts.Create(draft, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> Archive(BatchArchiveRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (items) =>
{
var batchModifyRequest = new BatchModifyMessagesRequest
{
Ids = items.Select(a => a.Item.Id.ToString()).ToList()
};
if (request.IsArchiving)
{
batchModifyRequest.RemoveLabelIds = new[] { GoogleIntegratorExtensions.INBOX_LABEL_ID };
}
else
{
batchModifyRequest.AddLabelIds = new[] { GoogleIntegratorExtensions.INBOX_LABEL_ID };
}
return _gmailService.Users.Messages.BatchModify(batchModifyRequest, "me");
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> SendDraft(BatchSendDraftRequestRequest request)
{
return CreateHttpBundle(request, (item) =>
{
if (item is not SendDraftRequest singleDraftRequest)
throw new ArgumentException("BatchSendDraftRequestRequest collection must be of type SendDraftRequest.");
var message = new Message();
if (!string.IsNullOrEmpty(singleDraftRequest.Item.ThreadId))
{
message.ThreadId = singleDraftRequest.Item.ThreadId;
}
singleDraftRequest.Request.Mime.Prepare(EncodingConstraint.None);
var mimeString = singleDraftRequest.Request.Mime.ToString();
var base64UrlEncodedMime = Base64UrlEncoder.Encode(mimeString);
message.Raw = base64UrlEncodedMime;
var draft = new Draft()
{
Id = singleDraftRequest.Request.MailItem.DraftId,
Message = message
};
return _gmailService.Users.Drafts.Send(draft, "me");
});
}
public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem,
ITransferProgress transferProgress = null,
CancellationToken cancellationToken = default)
{
var request = _gmailService.Users.Messages.Get("me", mailItem.Id);
request.Format = UsersResource.MessagesResource.GetRequest.FormatEnum.Raw;
var gmailMessage = await request.ExecuteAsync(cancellationToken).ConfigureAwait(false);
var mimeMessage = gmailMessage.GetGmailMimeMessage();
if (mimeMessage == null)
{
_logger.Warning("Tried to download Gmail Raw Mime with {Id} id and server responded without a data.", mailItem.Id);
return;
}
await _gmailChangeProcessor.SaveMimeFileAsync(mailItem.FileId, mimeMessage, Account.Id).ConfigureAwait(false);
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> RenameFolder(RenameFolderRequest request)
{
return CreateHttpBundleWithResponse<Label>(request, (item) =>
{
if (item is not RenameFolderRequest renameFolderRequest)
throw new ArgumentException($"Renaming folder must be handled with '{nameof(RenameFolderRequest)}'");
var label = new Label()
{
Name = renameFolderRequest.NewFolderName
};
return _gmailService.Users.Labels.Update(label, "me", request.Folder.RemoteFolderId);
});
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> EmptyFolder(EmptyFolderRequest request)
{
// Create batch delete request.
var deleteRequests = request.MailsToDelete.Select(a => new DeleteRequest(a));
return Delete(new BatchDeleteRequest(deleteRequests));
}
public override IEnumerable<IRequestBundle<IClientServiceRequest>> MarkFolderAsRead(MarkFolderAsReadRequest request)
=> MarkRead(new BatchMarkReadRequest(request.MailsToMarkRead.Select(a => new MarkReadRequest(a, true)), true));
#endregion
#region Request Execution
public override async Task ExecuteNativeRequestsAsync(IEnumerable<IRequestBundle<IClientServiceRequest>> batchedRequests,
CancellationToken cancellationToken = default)
{
var batchedBundles = batchedRequests.Batch((int)MaximumAllowedBatchRequestSize);
var bundleCount = batchedBundles.Count();
for (int i = 0; i < bundleCount; i++)
{
var bundle = batchedBundles.ElementAt(i);
var nativeBatchRequest = new BatchRequest(_gmailService);
var bundleRequestCount = bundle.Count();
for (int k = 0; k < bundleRequestCount; k++)
{
var requestBundle = bundle.ElementAt(k);
var nativeRequest = requestBundle.NativeRequest;
var request = requestBundle.Request;
request.ApplyUIChanges();
// TODO: Queue is synchronous. Create a task bucket to await all processing.
nativeBatchRequest.Queue<object>(nativeRequest, async (content, error, index, message)
=> await ProcessSingleNativeRequestResponseAsync(requestBundle, error, message, cancellationToken).ConfigureAwait(false));
}
await nativeBatchRequest.ExecuteAsync(cancellationToken).ConfigureAwait(false);
}
}
private void ProcessGmailRequestError(RequestError error)
{
if (error == null) return;
// OutOfMemoryException is a known bug in Gmail SDK.
if (error.Code == 0)
{
throw new OutOfMemoryException(error.Message);
}
// Entity not found.
if (error.Code == 404)
{
throw new SynchronizerEntityNotFoundException(error.Message);
}
if (!string.IsNullOrEmpty(error.Message))
{
error.Errors?.ForEach(error => _logger.Error("Unknown Gmail SDK error for {Name}\n{Error}", Account.Name, error));
// TODO: Debug
// throw new SynchronizerException(error.Message);
}
}
/// <summary>
/// Handles after each single message download.
/// This involves adding the Gmail message into Wino database.
/// </summary>
/// <param name="message"></param>
/// <param name="error"></param>
/// <param name="httpResponseMessage"></param>
/// <param name="cancellationToken"></param>
private async Task HandleSingleItemDownloadedCallbackAsync(Message message,
RequestError error,
string downloadingMessageId,
CancellationToken cancellationToken = default)
{
try
{
ProcessGmailRequestError(error);
}
catch (OutOfMemoryException)
{
_logger.Warning("Gmail SDK got OutOfMemoryException due to bug in the SDK");
}
catch (SynchronizerEntityNotFoundException)
{
_logger.Warning("Resource not found for {DownloadingMessageId}", downloadingMessageId);
}
catch (SynchronizerException synchronizerException)
{
_logger.Error("Gmail SDK returned error for {DownloadingMessageId}\n{SynchronizerException}", downloadingMessageId, synchronizerException);
}
if (message == null)
{
_logger.Warning("Skipped GMail message download for {DownloadingMessageId}", downloadingMessageId);
return;
}
// Gmail has LabelId property for each message.
// Therefore we can pass null as the assigned folder safely.
var mailPackage = await CreateNewMailPackagesAsync(message, null, cancellationToken);
// If CreateNewMailPackagesAsync returns null it means local draft mapping is done.
// We don't need to insert anything else.
if (mailPackage == null)
return;
foreach (var package in mailPackage)
{
await _gmailChangeProcessor.CreateMailAsync(Account.Id, package).ConfigureAwait(false);
}
// Try updating the history change identifier if any.
if (message.HistoryId == null) return;
// Delta changes also has history id but the maximum id is preserved in the account service.
// TODO: This is not good. Centralize the identifier fetch and prevent direct access here.
Account.SynchronizationDeltaIdentifier = await _gmailChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, message.HistoryId.ToString());
}
private async Task ProcessSingleNativeRequestResponseAsync(IRequestBundle<IClientServiceRequest> bundle,
RequestError error,
HttpResponseMessage httpResponseMessage,
CancellationToken cancellationToken = default)
{
ProcessGmailRequestError(error);
if (bundle is HttpRequestBundle<IClientServiceRequest, Message> messageBundle)
{
var gmailMessage = await messageBundle.DeserializeBundleAsync(httpResponseMessage, cancellationToken).ConfigureAwait(false);
if (gmailMessage == null) return;
await HandleSingleItemDownloadedCallbackAsync(gmailMessage, error, "unknown", cancellationToken);
}
else if (bundle is HttpRequestBundle<IClientServiceRequest, Label> folderBundle)
{
var gmailLabel = await folderBundle.DeserializeBundleAsync(httpResponseMessage, cancellationToken).ConfigureAwait(false);
if (gmailLabel == null) return;
// TODO: Handle new Gmail Label added or updated.
}
else if (bundle is HttpRequestBundle<IClientServiceRequest, Draft> draftBundle && draftBundle.Request is CreateDraftRequest createDraftRequest)
{
// New draft mail is created.
var messageDraft = await draftBundle.DeserializeBundleAsync(httpResponseMessage, cancellationToken).ConfigureAwait(false);
if (messageDraft == null) return;
var localDraftCopy = createDraftRequest.DraftPreperationRequest.CreatedLocalDraftCopy;
// Here we have DraftId, MessageId and ThreadId.
// Update the local copy properties and re-synchronize to get the original message and update history.
// We don't fetch the single message here because it may skip some of the history changes when the
// fetch updates the historyId. Therefore we need to re-synchronize to get the latest history changes
// which will have the original message downloaded eventually.
await _gmailChangeProcessor.MapLocalDraftAsync(Account.Id, localDraftCopy.UniqueId, messageDraft.Message.Id, messageDraft.Id, messageDraft.Message.ThreadId);
var options = new SynchronizationOptions()
{
AccountId = Account.Id,
Type = SynchronizationType.Full
};
await SynchronizeInternalAsync(options, cancellationToken);
}
}
/// <summary>
/// Maps existing Gmail Draft resources to local mail copies.
/// This uses indexed search, therefore it's quite fast.
/// It's safe to execute this after each Draft creation + batch message download.
/// </summary>
private async Task MapDraftIdsAsync(CancellationToken cancellationToken = default)
{
// TODO: This call is not necessary if we don't have any local drafts.
// Remote drafts will be downloaded in missing message batches anyways.
// Fix it by checking whether we need to do this or not.
var drafts = await _gmailService.Users.Drafts.List("me").ExecuteAsync(cancellationToken);
if (drafts.Drafts == null)
{
_logger.Information("There are no drafts to map for {Name}", Account.Name);
return;
}
foreach (var draft in drafts.Drafts)
{
await _gmailChangeProcessor.MapLocalDraftAsync(draft.Message.Id, draft.Id, draft.Message.ThreadId);
}
}
/// <summary>
/// Creates new mail packages for the given message.
/// AssignedFolder is null since the LabelId is parsed out of the Message.
/// </summary>
/// <param name="message">Gmail message to create package for.</param>
/// <param name="assignedFolder">Null, not used.</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>New mail package that change processor can use to insert new mail into database.</returns>
public override async Task<List<NewMailItemPackage>> CreateNewMailPackagesAsync(Message message,
MailItemFolder assignedFolder,
CancellationToken cancellationToken = default)
{
var packageList = new List<NewMailItemPackage>();
MimeMessage mimeMessage = message.GetGmailMimeMessage();
var mailCopy = message.AsMailCopy(mimeMessage);
// Check whether this message is mapped to any local draft.
// Previously we were using Draft resource response as mapping drafts.
// This seem to be a worse approach. Now both Outlook and Gmail use X-Wino-Draft-Id header to map drafts.
// This is a better approach since we don't need to fetch the draft resource to get the draft id.
if (mailCopy.IsDraft
&& mimeMessage.Headers.Contains(Constants.WinoLocalDraftHeader)
&& Guid.TryParse(mimeMessage.Headers[Constants.WinoLocalDraftHeader], out Guid localDraftCopyUniqueId))
{
// This message belongs to existing local draft copy.
// We don't need to create a new mail copy for this message, just update the existing one.
bool isMappingSuccesfull = await _gmailChangeProcessor.MapLocalDraftAsync(Account.Id, localDraftCopyUniqueId, mailCopy.Id, mailCopy.DraftId, mailCopy.ThreadId);
if (isMappingSuccesfull) return null;
// Local copy doesn't exists. Continue execution to insert mail copy.
}
if (message.LabelIds is not null)
{
foreach (var labelId in message.LabelIds)
{
packageList.Add(new NewMailItemPackage(mailCopy, mimeMessage, labelId));
}
}
return packageList;
}
#endregion
}
}

View File

@@ -0,0 +1,999 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MailKit;
using MailKit.Net.Imap;
using MailKit.Search;
using MimeKit;
using MoreLinq;
using Serilog;
using Wino.Core.Extensions;
using Wino.Core.Integration;
using Wino.Core.Mime;
using Wino.Domain;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Wino.Domain.Exceptions;
using Wino.Domain.Interfaces;
using Wino.Domain.Models.MailItem;
using Wino.Domain.Models.Synchronization;
using Wino.Services.Requests;
using Wino.Services.Requests.Bundles;
namespace Wino.Core.Synchronizers
{
public class ImapSynchronizer : BaseSynchronizer<ImapRequest, ImapMessageCreationPackage>
{
private CancellationTokenSource idleDoneToken;
private CancellationTokenSource cancelInboxListeningToken = new CancellationTokenSource();
private IMailFolder inboxFolder;
private readonly ILogger _logger = Log.ForContext<ImapSynchronizer>();
private readonly ImapClientPool _clientPool;
private readonly IImapChangeProcessor _imapChangeProcessor;
// Minimum summary items to Fetch for mail synchronization from IMAP.
private readonly MessageSummaryItems mailSynchronizationFlags =
MessageSummaryItems.Flags |
MessageSummaryItems.UniqueId |
MessageSummaryItems.ThreadId |
MessageSummaryItems.EmailId |
MessageSummaryItems.Headers |
MessageSummaryItems.PreviewText |
MessageSummaryItems.GMailThreadId |
MessageSummaryItems.References |
MessageSummaryItems.ModSeq;
/// <summary>
/// Timer that keeps the <see cref="InboxClient"/> alive for the lifetime of the pool.
/// Sends NOOP command to the server periodically.
/// </summary>
private Timer _noOpTimer;
/// <summary>
/// ImapClient that keeps the Inbox folder opened all the time for listening notifications.
/// </summary>
private ImapClient _inboxIdleClient;
public override uint BatchModificationSize => 1000;
public override uint InitialMessageDownloadCountPerFolder => 250;
public ImapSynchronizer(MailAccount account, IImapChangeProcessor imapChangeProcessor) : base(account)
{
_clientPool = new ImapClientPool(Account.ServerInformation);
_imapChangeProcessor = imapChangeProcessor;
idleDoneToken = new CancellationTokenSource();
}
// TODO
// private async void NoOpTimerTriggered(object state) => await AwaitInboxIdleAsync();
private async Task AwaitInboxIdleAsync()
{
if (_inboxIdleClient == null)
{
_logger.Warning("InboxClient is null. Cannot send NOOP command.");
return;
}
await _clientPool.EnsureConnectedAsync(_inboxIdleClient);
await _clientPool.EnsureAuthenticatedAsync(_inboxIdleClient);
try
{
if (inboxFolder == null)
{
inboxFolder = _inboxIdleClient.Inbox;
await inboxFolder.OpenAsync(FolderAccess.ReadOnly, cancelInboxListeningToken.Token);
}
idleDoneToken = new CancellationTokenSource();
await _inboxIdleClient.IdleAsync(idleDoneToken.Token, cancelInboxListeningToken.Token);
}
finally
{
idleDoneToken.Dispose();
idleDoneToken = null;
}
}
private async Task StopInboxListeningAsync()
{
if (inboxFolder != null)
{
inboxFolder.CountChanged -= InboxFolderCountChanged;
inboxFolder.MessageExpunged -= InboxFolderMessageExpunged;
inboxFolder.MessageFlagsChanged -= InboxFolderMessageFlagsChanged;
}
if (_noOpTimer != null)
{
_noOpTimer.Dispose();
_noOpTimer = null;
}
if (idleDoneToken != null)
{
idleDoneToken.Cancel();
idleDoneToken.Dispose();
idleDoneToken = null;
}
if (_inboxIdleClient != null)
{
await _inboxIdleClient.DisconnectAsync(true);
_inboxIdleClient.Dispose();
_inboxIdleClient = null;
}
}
/// <summary>
/// Tries to connect & authenticate with the given credentials.
/// Prepares synchronizer for active listening of Inbox folder.
/// </summary>
public async Task StartInboxListeningAsync()
{
_inboxIdleClient = await _clientPool.GetClientAsync();
// Run it every 8 minutes after 1 minute delay.
// _noOpTimer = new Timer(NoOpTimerTriggered, null, 60000, 8 * 60 * 1000);
await _clientPool.EnsureConnectedAsync(_inboxIdleClient);
await _clientPool.EnsureAuthenticatedAsync(_inboxIdleClient);
if (!_inboxIdleClient.Capabilities.HasFlag(ImapCapabilities.Idle))
{
_logger.Information("Imap server does not support IDLE command. Listening live changes is not supported for {Name}", Account.Name);
return;
}
inboxFolder = _inboxIdleClient.Inbox;
if (inboxFolder == null)
{
_logger.Information("Inbox folder is null. Cannot listen for changes.");
return;
}
inboxFolder.CountChanged += InboxFolderCountChanged;
inboxFolder.MessageExpunged += InboxFolderMessageExpunged;
inboxFolder.MessageFlagsChanged += InboxFolderMessageFlagsChanged;
while (!cancelInboxListeningToken.IsCancellationRequested)
{
await AwaitInboxIdleAsync();
}
await StopInboxListeningAsync();
}
private void InboxFolderMessageFlagsChanged(object sender, MessageFlagsChangedEventArgs e)
{
Console.WriteLine("Flags have changed for message #{0} ({1}).", e.Index, e.Flags);
}
private void InboxFolderMessageExpunged(object sender, MessageEventArgs e)
{
_logger.Information("Inbox folder message expunged");
}
private void InboxFolderCountChanged(object sender, EventArgs e)
{
_logger.Information("Inbox folder count changed.");
}
/// <summary>
/// Parses List of string of mail copy ids and return valid uIds.
/// Follow the rules for creating arbitrary unique id for mail copies.
/// </summary>
private UniqueIdSet GetUniqueIds(IEnumerable<string> mailCopyIds)
=> new(mailCopyIds.Select(a => new UniqueId(Wino.Domain.Extensions.MailkitClientExtensions.ResolveUid(a))));
#region Mail Integrations
// Items are grouped before being passed to this method.
// Meaning that all items will come from and to the same folder.
// It's fine to assume that here.
public override IEnumerable<IRequestBundle<ImapRequest>> Move(BatchMoveRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var uniqueIds = GetUniqueIds(request.Items.Select(a => a.Item.Id));
var sourceFolder = await client.GetFolderAsync(request.FromFolder.RemoteFolderId);
var destinationFolder = await client.GetFolderAsync(request.ToFolder.RemoteFolderId);
// Only opening source folder is enough.
await sourceFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false);
await sourceFolder.MoveToAsync(uniqueIds, destinationFolder).ConfigureAwait(false);
await sourceFolder.CloseAsync().ConfigureAwait(false);
}, request);
}
public override IEnumerable<IRequestBundle<ImapRequest>> ChangeFlag(BatchChangeFlagRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var folder = request.Items.First().Item.AssignedFolder;
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId);
var uniqueIds = GetUniqueIds(request.Items.Select(a => a.Item.Id));
await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false);
await remoteFolder.StoreAsync(uniqueIds, new StoreFlagsRequest(request.IsFlagged ? StoreAction.Add : StoreAction.Remove, MessageFlags.Flagged) { Silent = true }).ConfigureAwait(false);
await remoteFolder.CloseAsync().ConfigureAwait(false);
}, request);
}
public override IEnumerable<IRequestBundle<ImapRequest>> Delete(BatchDeleteRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var folder = request.Items.First().Item.AssignedFolder;
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId).ConfigureAwait(false);
var uniqueIds = GetUniqueIds(request.Items.Select(a => a.Item.Id));
await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false);
await remoteFolder.StoreAsync(uniqueIds, new StoreFlagsRequest(StoreAction.Add, MessageFlags.Deleted) { Silent = true }).ConfigureAwait(false);
await remoteFolder.ExpungeAsync().ConfigureAwait(false);
await remoteFolder.CloseAsync().ConfigureAwait(false);
}, request);
}
public override IEnumerable<IRequestBundle<ImapRequest>> MarkRead(BatchMarkReadRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var folder = request.Items.First().Item.AssignedFolder;
var remoteFolder = await client.GetFolderAsync(folder.RemoteFolderId);
var uniqueIds = GetUniqueIds(request.Items.Select(a => a.Item.Id));
await remoteFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false);
await remoteFolder.StoreAsync(uniqueIds, new StoreFlagsRequest(request.IsRead ? StoreAction.Add : StoreAction.Remove, MessageFlags.Seen) { Silent = true }).ConfigureAwait(false);
await remoteFolder.CloseAsync().ConfigureAwait(false);
}, request);
}
public override IEnumerable<IRequestBundle<ImapRequest>> CreateDraft(BatchCreateDraftRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var remoteDraftFolder = await client.GetFolderAsync(request.DraftPreperationRequest.CreatedLocalDraftCopy.AssignedFolder.RemoteFolderId).ConfigureAwait(false);
await remoteDraftFolder.OpenAsync(FolderAccess.ReadWrite).ConfigureAwait(false);
await remoteDraftFolder.AppendAsync(request.DraftPreperationRequest.CreatedLocalDraftMimeMessage, MessageFlags.Draft).ConfigureAwait(false);
await remoteDraftFolder.CloseAsync().ConfigureAwait(false);
}, request);
}
public override IEnumerable<IRequestBundle<ImapRequest>> Archive(BatchArchiveRequest request)
=> Move(new BatchMoveRequest(request.Items, request.FromFolder, request.ToFolder));
public override IEnumerable<IRequestBundle<ImapRequest>> EmptyFolder(EmptyFolderRequest request)
=> Delete(new BatchDeleteRequest(request.MailsToDelete.Select(a => new DeleteRequest(a))));
public override IEnumerable<IRequestBundle<ImapRequest>> MarkFolderAsRead(MarkFolderAsReadRequest request)
=> MarkRead(new BatchMarkReadRequest(request.MailsToMarkRead.Select(a => new MarkReadRequest(a, true)), true));
public override IEnumerable<IRequestBundle<ImapRequest>> SendDraft(BatchSendDraftRequestRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
// Batch sending is not supported. It will always be a single request therefore no need for a loop here.
var singleRequest = request.Request;
singleRequest.Mime.Prepare(EncodingConstraint.None);
using var smtpClient = new MailKit.Net.Smtp.SmtpClient();
if (smtpClient.IsConnected && client.IsAuthenticated) return;
if (!smtpClient.IsConnected)
await smtpClient.ConnectAsync(Account.ServerInformation.OutgoingServer, int.Parse(Account.ServerInformation.OutgoingServerPort), MailKit.Security.SecureSocketOptions.Auto);
if (!smtpClient.IsAuthenticated)
await smtpClient.AuthenticateAsync(Account.ServerInformation.OutgoingServerUsername, Account.ServerInformation.OutgoingServerPassword);
// TODO: Transfer progress implementation as popup in the UI.
await smtpClient.SendAsync(singleRequest.Mime, default);
await smtpClient.DisconnectAsync(true);
// SMTP sent the message, but we need to remove it from the Draft folder.
var draftFolder = singleRequest.MailItem.AssignedFolder;
var folder = await client.GetFolderAsync(draftFolder.RemoteFolderId);
await folder.OpenAsync(FolderAccess.ReadWrite);
var notUpdatedIds = await folder.StoreAsync(new UniqueId(Wino.Domain.Extensions.MailkitClientExtensions.ResolveUid(singleRequest.MailItem.Id)), new StoreFlagsRequest(StoreAction.Add, MessageFlags.Deleted) { Silent = true });
await folder.ExpungeAsync();
await folder.CloseAsync();
// Check whether we need to create a copy of the message to Sent folder.
// This comes from the account preferences.
if (singleRequest.AccountPreferences.ShouldAppendMessagesToSentFolder && singleRequest.SentFolder != null)
{
var sentFolder = await client.GetFolderAsync(singleRequest.SentFolder.RemoteFolderId);
await sentFolder.OpenAsync(FolderAccess.ReadWrite);
// Delete local Wino draft header. Otherwise mapping will be applied on re-sync.
singleRequest.Mime.Headers.Remove(Constants.WinoLocalDraftHeader);
await sentFolder.AppendAsync(singleRequest.Mime, MessageFlags.Seen);
await sentFolder.CloseAsync();
}
}, request);
}
public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem,
ITransferProgress transferProgress = null,
CancellationToken cancellationToken = default)
{
var folder = mailItem.AssignedFolder;
var remoteFolderId = folder.RemoteFolderId;
var client = await _clientPool.GetClientAsync().ConfigureAwait(false);
var remoteFolder = await client.GetFolderAsync(remoteFolderId, cancellationToken).ConfigureAwait(false);
var uniqueId = new UniqueId(Wino.Domain.Extensions.MailkitClientExtensions.ResolveUid(mailItem.Id));
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
var message = await remoteFolder.GetMessageAsync(uniqueId, cancellationToken, transferProgress).ConfigureAwait(false);
await _imapChangeProcessor.SaveMimeFileAsync(mailItem.FileId, message, Account.Id).ConfigureAwait(false);
await remoteFolder.CloseAsync(false, cancellationToken).ConfigureAwait(false);
_clientPool.Release(client);
}
public override IEnumerable<IRequestBundle<ImapRequest>> RenameFolder(RenameFolderRequest request)
{
return CreateTaskBundle(async (ImapClient client) =>
{
var folder = await client.GetFolderAsync(request.Folder.RemoteFolderId).ConfigureAwait(false);
await folder.RenameAsync(folder.ParentFolder, request.NewFolderName).ConfigureAwait(false);
}, request);
}
#endregion
public override async Task<List<NewMailItemPackage>> CreateNewMailPackagesAsync(ImapMessageCreationPackage message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default)
{
var imapFolder = message.MailFolder;
var summary = message.MessageSummary;
var mimeMessage = await imapFolder.GetMessageAsync(summary.UniqueId, cancellationToken).ConfigureAwait(false);
var mailCopy = summary.GetMailDetails(assignedFolder, mimeMessage);
// Draft folder message updates must be updated as IsDraft.
// I couldn't find it in MimeMessage...
mailCopy.IsDraft = assignedFolder.SpecialFolderType == SpecialFolderType.Draft;
// Check draft mapping.
// This is the same implementation as in the OutlookSynchronizer.
if (mimeMessage.Headers.Contains(Constants.WinoLocalDraftHeader)
&& Guid.TryParse(mimeMessage.Headers[Constants.WinoLocalDraftHeader], out Guid localDraftCopyUniqueId))
{
// This message belongs to existing local draft copy.
// We don't need to create a new mail copy for this message, just update the existing one.
bool isMappingSuccessful = await _imapChangeProcessor.MapLocalDraftAsync(Account.Id, localDraftCopyUniqueId, mailCopy.Id, mailCopy.DraftId, mailCopy.ThreadId);
if (isMappingSuccessful) return null;
// Local copy doesn't exists. Continue execution to insert mail copy.
}
var package = new NewMailItemPackage(mailCopy, mimeMessage, assignedFolder.RemoteFolderId);
return
[
package
];
}
public override async Task<SynchronizationResult> SynchronizeInternalAsync(SynchronizationOptions options, CancellationToken cancellationToken = default)
{
// options.Type = SynchronizationType.FoldersOnly;
var downloadedMessageIds = new List<string>();
_logger.Information("Internal synchronization started for {Name}", Account.Name);
_logger.Information("Options: {Options}", options);
options.ProgressListener?.AccountProgressUpdated(Account.Id, 1);
// Only do folder sync for these types.
// Opening folder and checking their UidValidity is slow.
// Therefore this should be avoided as many times as possible.
// This may create some inconsistencies, but nothing we can do...
await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false);
if (options.Type != SynchronizationType.FoldersOnly)
{
var synchronizationFolders = await _imapChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false);
for (int i = 0; i < synchronizationFolders.Count; i++)
{
var folder = synchronizationFolders[i];
var progress = (int)Math.Round((double)(i + 1) / synchronizationFolders.Count * 100);
options.ProgressListener?.AccountProgressUpdated(Account.Id, progress);
var folderDownloadedMessageIds = await SynchronizeFolderInternalAsync(folder, cancellationToken).ConfigureAwait(false);
downloadedMessageIds.AddRange(folderDownloadedMessageIds);
}
}
options.ProgressListener?.AccountProgressUpdated(Account.Id, 100);
// Get all unread new downloaded items and return in the result.
// This is primarily used in notifications.
var unreadNewItems = await _imapChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false);
return SynchronizationResult.Completed(unreadNewItems);
}
public override async Task ExecuteNativeRequestsAsync(IEnumerable<IRequestBundle<ImapRequest>> batchedRequests, CancellationToken cancellationToken = default)
{
// First apply the UI changes for each bundle.
// This is important to reflect changes to the UI before the network call is done.
foreach (var item in batchedRequests)
{
item.Request.ApplyUIChanges();
}
// All task bundles will execute on the same client.
// Tasks themselves don't pull the client from the pool
// because exception handling is easier this way.
// Also we might parallelize these bundles later on for additional performance.
foreach (var item in batchedRequests)
{
// At this point this client is ready to execute async commands.
// Each task bundle will await and execution will continue in case of error.
ImapClient executorClient = null;
bool isCrashed = false;
try
{
executorClient = await _clientPool.GetClientAsync();
}
catch (ImapClientPoolException)
{
// Client pool failed to get a client.
// Requests may not be executed at this point.
item.Request.RevertUIChanges();
isCrashed = true;
throw;
}
finally
{
// Make sure that the client is released from the pool for next usages if error occurs.
if (isCrashed && executorClient != null)
{
_clientPool.Release(executorClient);
}
}
// TODO: Retry pattern.
// TODO: Error handling.
try
{
await item.NativeRequest.IntegratorTask(executorClient).ConfigureAwait(false);
}
catch (Exception)
{
item.Request.RevertUIChanges();
throw;
}
finally
{
_clientPool.Release(executorClient);
}
}
}
/// <summary>
/// Assigns special folder type for the given local folder.
/// If server doesn't support special folders, we can't determine the type. MailKit will throw for GetFolder.
/// Default type is Other.
/// </summary>
/// <param name="executorClient">ImapClient from the pool</param>
/// <param name="remoteFolder">Assigning remote folder.</param>
/// <param name="localFolder">Assigning local folder.</param>
private void AssignSpecialFolderType(ImapClient executorClient, IMailFolder remoteFolder, MailItemFolder localFolder)
{
// Inbox is awlawys available. Don't miss it for assignment even though XList or SpecialUser is not supported.
if (executorClient.Inbox == remoteFolder)
{
localFolder.SpecialFolderType = SpecialFolderType.Inbox;
return;
}
bool isSpecialFoldersSupported = executorClient.Capabilities.HasFlag(ImapCapabilities.SpecialUse) || executorClient.Capabilities.HasFlag(ImapCapabilities.XList);
if (!isSpecialFoldersSupported)
{
localFolder.SpecialFolderType = SpecialFolderType.Other;
return;
}
if (remoteFolder == executorClient.Inbox)
localFolder.SpecialFolderType = SpecialFolderType.Inbox;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Drafts))
localFolder.SpecialFolderType = SpecialFolderType.Draft;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Junk))
localFolder.SpecialFolderType = SpecialFolderType.Junk;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Trash))
localFolder.SpecialFolderType = SpecialFolderType.Deleted;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Sent))
localFolder.SpecialFolderType = SpecialFolderType.Sent;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Archive))
localFolder.SpecialFolderType = SpecialFolderType.Archive;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Important))
localFolder.SpecialFolderType = SpecialFolderType.Important;
else if (remoteFolder == executorClient.GetFolder(SpecialFolder.Flagged))
localFolder.SpecialFolderType = SpecialFolderType.Starred;
}
private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default)
{
// https://www.rfc-editor.org/rfc/rfc4549#section-1.1
var localFolders = await _imapChangeProcessor.GetLocalFoldersAsync(Account.Id).ConfigureAwait(false);
ImapClient executorClient = null;
try
{
List<MailItemFolder> insertedFolders = new();
List<MailItemFolder> updatedFolders = new();
List<MailItemFolder> deletedFolders = new();
executorClient = await _clientPool.GetClientAsync().ConfigureAwait(false);
var remoteFolders = (await executorClient.GetFoldersAsync(executorClient.PersonalNamespaces[0], cancellationToken: cancellationToken)).ToList();
// 1. First check deleted folders.
// 1.a If local folder doesn't exists remotely, delete it.
// 1.b If local folder exists remotely, check if it is still a valid folder. If UidValidity is changed, delete it.
foreach (var localFolder in localFolders)
{
IMailFolder remoteFolder = null;
try
{
remoteFolder = remoteFolders.FirstOrDefault(a => a.FullName == localFolder.RemoteFolderId);
bool shouldDeleteLocalFolder = false;
// Check UidValidity of the remote folder if exists.
if (remoteFolder != null)
{
// UidValidity won't be available until it's opened.
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken).ConfigureAwait(false);
shouldDeleteLocalFolder = remoteFolder.UidValidity != localFolder.UidValidity;
}
else
{
// Remote folder doesn't exist. Delete it.
shouldDeleteLocalFolder = true;
}
if (shouldDeleteLocalFolder)
{
await _imapChangeProcessor.DeleteFolderAsync(Account.Id, localFolder.RemoteFolderId).ConfigureAwait(false);
deletedFolders.Add(localFolder);
}
}
catch (Exception)
{
throw;
}
finally
{
if (remoteFolder != null)
{
await remoteFolder.CloseAsync().ConfigureAwait(false);
}
}
}
deletedFolders.ForEach(a => localFolders.Remove(a));
// 2. Get all remote folders and insert/update each of them.
var nameSpace = executorClient.PersonalNamespaces[0];
IMailFolder inbox = executorClient.Inbox;
// Sometimes Inbox is the root namespace. We need to check for that.
if (inbox != null && !remoteFolders.Contains(inbox))
remoteFolders.Add(inbox);
foreach (var remoteFolder in remoteFolders)
{
// Namespaces are not needed as folders.
// Non-existed folders don't need to be synchronized.
if ((remoteFolder.IsNamespace && !remoteFolder.Attributes.HasFlag(FolderAttributes.Inbox)) || !remoteFolder.Exists)
continue;
var existingLocalFolder = localFolders.FirstOrDefault(a => a.RemoteFolderId == remoteFolder.FullName);
if (existingLocalFolder == null)
{
// Folder doesn't exist locally. Insert it.
var localFolder = remoteFolder.GetLocalFolder();
// Check whether this is a special folder.
AssignSpecialFolderType(executorClient, remoteFolder, localFolder);
bool isSystemFolder = localFolder.SpecialFolderType != SpecialFolderType.Other;
localFolder.IsSynchronizationEnabled = isSystemFolder;
localFolder.IsSticky = isSystemFolder;
// By default, all special folders update unread count in the UI except Trash.
localFolder.ShowUnreadCount = localFolder.SpecialFolderType != SpecialFolderType.Deleted || localFolder.SpecialFolderType != SpecialFolderType.Other;
localFolder.MailAccountId = Account.Id;
// Sometimes sub folders are parented under Inbox.
// Even though this makes sense in server level, in the client it sucks.
// That will make sub folders to be parented under Inbox in the client.
// Instead, we will mark them as non-parented folders.
// This is better. Model allows personalized folder structure anyways
// even though we don't have the page/control to adjust it.
if (remoteFolder.ParentFolder == executorClient.Inbox)
localFolder.ParentRemoteFolderId = string.Empty;
// Set UidValidity for cache expiration.
// Folder must be opened for this.
await remoteFolder.OpenAsync(FolderAccess.ReadOnly, cancellationToken);
localFolder.UidValidity = remoteFolder.UidValidity;
await remoteFolder.CloseAsync(cancellationToken: cancellationToken);
insertedFolders.Add(localFolder);
}
else
{
// Update existing folder. Right now we only update the name.
// TODO: Moving folders around different parents. This is not supported right now.
// We will need more comphrensive folder update mechanism to support this.
if (ShouldUpdateFolder(remoteFolder, existingLocalFolder))
{
existingLocalFolder.FolderName = remoteFolder.Name;
updatedFolders.Add(existingLocalFolder);
}
else
{
// Remove it from the local folder list to skip additional folder updates.
localFolders.Remove(existingLocalFolder);
}
}
}
// Process changes in order-> Insert, Update. Deleted ones are already processed.
foreach (var folder in insertedFolders)
{
await _imapChangeProcessor.InsertFolderAsync(folder).ConfigureAwait(false);
}
foreach (var folder in updatedFolders)
{
await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false);
}
}
catch (Exception ex)
{
_logger.Error(ex, "Synchronizing IMAP folders failed.");
throw;
}
finally
{
if (executorClient != null)
{
_clientPool.Release(executorClient);
}
}
}
private async Task<IEnumerable<string>> SynchronizeFolderInternalAsync(MailItemFolder folder, CancellationToken cancellationToken = default)
{
if (!folder.IsSynchronizationEnabled) return default;
var downloadedMessageIds = new List<string>();
// STEP1: Ask for flag changes for older mails.
// STEP2: Get new mail changes.
// https://www.rfc-editor.org/rfc/rfc4549 - Section 4.3
var _synchronizationClient = await _clientPool.GetClientAsync();
IMailFolder imapFolder = null;
var knownMailIds = new UniqueIdSet();
var locallyKnownMailUids = await _imapChangeProcessor.GetKnownUidsForFolderAsync(folder.Id);
knownMailIds.AddRange(locallyKnownMailUids.Select(a => new UniqueId(a)));
var highestUniqueId = Math.Max(0, locallyKnownMailUids.Count == 0 ? 0 : locallyKnownMailUids.Max());
var missingMailIds = new UniqueIdSet();
var uidValidity = folder.UidValidity;
var highestModeSeq = folder.HighestModeSeq;
var logger = Log.ForContext("FolderName", folder.FolderName);
logger.Verbose("HighestModeSeq: {HighestModeSeq}, HighestUniqueId: {HighestUniqueId}, UIDValidity: {UIDValidity}", highestModeSeq, highestUniqueId, uidValidity);
// Event handlers are placed here to handle existing MailItemFolder and IIMailFolder from MailKit.
// MailKit doesn't expose folder data when these events are emitted.
// Use local folder's UidValidty because cache might've been expired for remote IMAP folder.
// That will make our mail copy id invalid.
EventHandler<MessagesVanishedEventArgs> MessageVanishedHandler = async (s, e) =>
{
if (imapFolder == null) return;
foreach (var uniqueId in e.UniqueIds)
{
var localMailCopyId = Extensions.MailkitClientExtensions.CreateUid(folder.Id, uniqueId.Id);
await _imapChangeProcessor.DeleteMailAsync(Account.Id, localMailCopyId);
}
};
EventHandler<MessageFlagsChangedEventArgs> MessageFlagsChangedHandler = async (s, e) =>
{
if (imapFolder == null) return;
var localMailCopyId = Extensions.MailkitClientExtensions.CreateUid(folder.Id, e.UniqueId.Value.Id);
var isFlagged = Extensions.MailkitClientExtensions.GetIsFlagged(e.Flags);
var isRead = Extensions.MailkitClientExtensions.GetIsRead(e.Flags);
await _imapChangeProcessor.ChangeMailReadStatusAsync(localMailCopyId, isRead);
await _imapChangeProcessor.ChangeFlagStatusAsync(localMailCopyId, isFlagged);
};
EventHandler<MessageEventArgs> MessageExpungedHandler = async (s, e) =>
{
if (imapFolder == null) return;
if (e.UniqueId == null) return;
var localMailCopyId = Extensions.MailkitClientExtensions.CreateUid(folder.Id, e.UniqueId.Value.Id);
await _imapChangeProcessor.DeleteMailAsync(Account.Id, localMailCopyId);
};
try
{
imapFolder = await _synchronizationClient.GetFolderAsync(folder.RemoteFolderId, cancellationToken);
imapFolder.MessageFlagsChanged += MessageFlagsChangedHandler;
// TODO: Bug: Enabling quick re-sync actually doesn't enable it.
var qsyncEnabled = false; // _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.QuickResync);
var condStoreEnabled = _synchronizationClient.Capabilities.HasFlag(ImapCapabilities.CondStore);
if (qsyncEnabled)
{
imapFolder.MessagesVanished += MessageVanishedHandler;
await imapFolder.OpenAsync(FolderAccess.ReadWrite, uidValidity, (ulong)highestModeSeq, knownMailIds, cancellationToken);
// Check the folder validity.
// We'll delete our existing cache if it's not.
// Get all messages after the last successful synchronization date.
// This is fine for Wino synchronization because we're not really looking to
// synchronize all folder.
var allMessageIds = await imapFolder.SearchAsync(SearchQuery.All, cancellationToken);
if (uidValidity != imapFolder.UidValidity)
{
// TODO: Cache is invalid. Delete all local cache.
//await ChangeProcessor.FolderService.ClearImapFolderCacheAsync(folder.Id);
folder.UidValidity = imapFolder.UidValidity;
missingMailIds.AddRange(allMessageIds);
}
else
{
// Cache is valid.
// Add missing mails only.
missingMailIds.AddRange(allMessageIds.Except(knownMailIds).Where(a => a.Id > highestUniqueId));
}
}
else
{
// QSYNC extension is not enabled for the server.
// We rely on ConditionalStore.
imapFolder.MessageExpunged += MessageExpungedHandler;
await imapFolder.OpenAsync(FolderAccess.ReadWrite, cancellationToken);
// Get all messages after the last succesful synchronization date.
// This is fine for Wino synchronization because we're not really looking to
// synchronize all folder.
var allMessageIds = await imapFolder.SearchAsync(SearchQuery.All, cancellationToken);
if (uidValidity != imapFolder.UidValidity)
{
// TODO: Cache is invalid. Delete all local cache.
// await ChangeProcessor.FolderService.ClearImapFolderCacheAsync(folder.Id);
folder.UidValidity = imapFolder.UidValidity;
missingMailIds.AddRange(allMessageIds);
}
else
{
// Cache is valid.
var purgedMessages = knownMailIds.Except(allMessageIds);
foreach (var purgedMessage in purgedMessages)
{
var mailId = Extensions.MailkitClientExtensions.CreateUid(folder.Id, purgedMessage.Id);
await _imapChangeProcessor.DeleteMailAsync(Account.Id, mailId);
}
IList<IMessageSummary> changed;
if (knownMailIds.Count > 0)
{
// CONDSTORE enabled. Fetch items with highest mode seq for known items
// to track flag changes. Otherwise just get changes without the mode seq.
if (condStoreEnabled)
changed = await imapFolder.FetchAsync(knownMailIds, (ulong)highestModeSeq, MessageSummaryItems.Flags | MessageSummaryItems.ModSeq | MessageSummaryItems.UniqueId);
else
changed = await imapFolder.FetchAsync(knownMailIds, MessageSummaryItems.Flags | MessageSummaryItems.UniqueId);
foreach (var changedItem in changed)
{
var localMailCopyId = Extensions.MailkitClientExtensions.CreateUid(folder.Id, changedItem.UniqueId.Id);
var isFlagged = Extensions.MailkitClientExtensions.GetIsFlagged(changedItem.Flags);
var isRead = Extensions.MailkitClientExtensions.GetIsRead(changedItem.Flags);
await _imapChangeProcessor.ChangeMailReadStatusAsync(localMailCopyId, isRead);
await _imapChangeProcessor.ChangeFlagStatusAsync(localMailCopyId, isFlagged);
}
}
// We're only interested in items that has highier known uid than we fetched before.
// Others are just older messages.
missingMailIds.AddRange(allMessageIds.Except(knownMailIds).Where(a => a.Id > highestUniqueId));
}
}
// Fetch completely missing new items in the end.
// Limit check.
if (missingMailIds.Count > InitialMessageDownloadCountPerFolder)
{
missingMailIds = new UniqueIdSet(missingMailIds.TakeLast((int)InitialMessageDownloadCountPerFolder));
}
// In case of the high input, we'll batch them by 50 to reflect changes quickly.
var batchedMissingMailIds = missingMailIds.Batch(50).Select(a => new UniqueIdSet(a, SortOrder.Descending));
foreach (var batchMissingMailIds in batchedMissingMailIds)
{
var summaries = await imapFolder.FetchAsync(batchMissingMailIds, mailSynchronizationFlags, cancellationToken).ConfigureAwait(false);
foreach (var summary in summaries)
{
// We pass the opened folder and summary to retrieve raw MimeMessage.
var creationPackage = new ImapMessageCreationPackage(summary, imapFolder);
var createdMailPackages = await CreateNewMailPackagesAsync(creationPackage, folder, cancellationToken).ConfigureAwait(false);
// Local draft is mapped. We don't need to create a new mail copy.
if (createdMailPackages == null)
continue;
foreach (var mailPackage in createdMailPackages)
{
await _imapChangeProcessor.CreateMailAsync(Account.Id, mailPackage).ConfigureAwait(false);
}
}
}
if (folder.HighestModeSeq != (long)imapFolder.HighestModSeq)
{
folder.HighestModeSeq = (long)imapFolder.HighestModSeq;
await _imapChangeProcessor.UpdateFolderAsync(folder).ConfigureAwait(false);
}
// Update last synchronization date for the folder..
await _imapChangeProcessor.UpdateFolderLastSyncDateAsync(folder.Id).ConfigureAwait(false);
return downloadedMessageIds;
}
catch (FolderNotFoundException)
{
await _imapChangeProcessor.DeleteFolderAsync(Account.Id, folder.RemoteFolderId).ConfigureAwait(false);
return default;
}
catch (Exception)
{
throw;
}
finally
{
if (imapFolder != null)
{
imapFolder.MessageFlagsChanged -= MessageFlagsChangedHandler;
imapFolder.MessageExpunged -= MessageExpungedHandler;
imapFolder.MessagesVanished -= MessageVanishedHandler;
if (imapFolder.IsOpen)
await imapFolder.CloseAsync();
}
_clientPool.Release(_synchronizationClient);
}
}
/// <summary>
/// Whether the local folder should be updated with the remote folder.
/// IMAP only compares folder name for now.
/// </summary>
/// <param name="remoteFolder">Remote folder</param>
/// <param name="localFolder">Local folder.</param>
public bool ShouldUpdateFolder(IMailFolder remoteFolder, MailItemFolder localFolder) => remoteFolder.Name != localFolder.FolderName;
}
}

View File

@@ -0,0 +1,753 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Graph;
using Microsoft.Graph.Models;
using Microsoft.Kiota.Abstractions;
using Microsoft.Kiota.Abstractions.Authentication;
using MimeKit;
using MoreLinq.Extensions;
using Serilog;
using Wino.Domain;
using Wino.Domain.Exceptions;
using Wino.Core.Extensions;
using Wino.Core.Http;
using Wino.Domain.Entities;
using Wino.Domain.Enums;
using Wino.Domain.Interfaces;
using Wino.Domain.Models.MailItem;
using Wino.Domain.Models.Synchronization;
using Wino.Services.Requests;
using Wino.Services.Requests.Bundles;
namespace Wino.Core.Synchronizers
{
public class OutlookSynchronizer : BaseSynchronizer<RequestInformation, Message>
{
public override uint BatchModificationSize => 20;
public override uint InitialMessageDownloadCountPerFolder => 250;
private const uint MaximumAllowedBatchRequestSize = 20;
private const string INBOX_NAME = "inbox";
private const string SENT_NAME = "sentitems";
private const string DELETED_NAME = "deleteditems";
private const string JUNK_NAME = "junkemail";
private const string DRAFTS_NAME = "drafts";
private const string ARCHIVE_NAME = "archive";
private readonly string[] outlookMessageSelectParameters =
[
"InferenceClassification",
"Flag",
"Importance",
"IsRead",
"IsDraft",
"ReceivedDateTime",
"HasAttachments",
"BodyPreview",
"Id",
"ConversationId",
"From",
"Subject",
"ParentFolderId",
"InternetMessageId",
];
private readonly SemaphoreSlim _handleItemRetrievalSemaphore = new(1);
private readonly ILogger _logger = Log.ForContext<OutlookSynchronizer>();
private readonly IOutlookChangeProcessor _outlookChangeProcessor;
private readonly GraphServiceClient _graphClient;
public OutlookSynchronizer(MailAccount account,
IAuthenticator authenticator,
IOutlookChangeProcessor outlookChangeProcessor) : base(account)
{
var tokenProvider = new MicrosoftTokenProvider(Account, authenticator);
// Add immutable id preffered client.
var handlers = GraphClientFactory.CreateDefaultHandlers();
handlers.Add(new MicrosoftImmutableIdHandler());
var httpClient = GraphClientFactory.Create(handlers);
_graphClient = new GraphServiceClient(httpClient, new BaseBearerTokenAuthenticationProvider(tokenProvider));
_outlookChangeProcessor = outlookChangeProcessor;
// Specify to use TLS 1.2 as default connection
System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
}
public override async Task<SynchronizationResult> SynchronizeInternalAsync(SynchronizationOptions options, CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List<string>();
_logger.Information("Internal synchronization started for {Name}", Account.Name);
_logger.Information("Options: {Options}", options);
try
{
options.ProgressListener?.AccountProgressUpdated(Account.Id, 1);
await SynchronizeFoldersAsync(cancellationToken).ConfigureAwait(false);
if (options.Type != SynchronizationType.FoldersOnly)
{
var synchronizationFolders = await _outlookChangeProcessor.GetSynchronizationFoldersAsync(options).ConfigureAwait(false);
_logger.Information("Found {Count} folders to synchronize.", synchronizationFolders.Count);
_logger.Information(string.Format("Folders: {0}", string.Join(",", synchronizationFolders.Select(a => a.FolderName))));
for (int i = 0; i < synchronizationFolders.Count; i++)
{
var folder = synchronizationFolders[i];
var progress = (int)Math.Round((double)(i + 1) / synchronizationFolders.Count * 100);
options.ProgressListener?.AccountProgressUpdated(Account.Id, progress);
var folderDownloadedMessageIds = await SynchronizeFolderAsync(folder, cancellationToken).ConfigureAwait(false);
downloadedMessageIds.AddRange(folderDownloadedMessageIds);
}
}
}
catch (Exception ex)
{
_logger.Error(ex, "Synchronization failed for {Name}", Account.Name);
throw;
}
finally
{
options.ProgressListener?.AccountProgressUpdated(Account.Id, 100);
}
// Get all unred new downloaded items and return in the result.
// This is primarily used in notifications.
var unreadNewItems = await _outlookChangeProcessor.GetDownloadedUnreadMailsAsync(Account.Id, downloadedMessageIds).ConfigureAwait(false);
return SynchronizationResult.Completed(unreadNewItems);
}
private async Task<IEnumerable<string>> SynchronizeFolderAsync(MailItemFolder folder, CancellationToken cancellationToken = default)
{
var downloadedMessageIds = new List<string>();
_logger.Debug("Started synchronization for folder {FolderName}", folder.FolderName);
cancellationToken.ThrowIfCancellationRequested();
string latestDeltaLink = string.Empty;
bool isInitialSync = string.IsNullOrEmpty(folder.DeltaToken);
Microsoft.Graph.Me.MailFolders.Item.Messages.Delta.DeltaGetResponse messageCollectionPage = null;
if (isInitialSync)
{
_logger.Debug("No sync identifier for Folder {FolderName}. Performing initial sync.", folder.FolderName);
// No delta link. Performing initial sync.
messageCollectionPage = await _graphClient.Me.MailFolders[folder.RemoteFolderId].Messages.Delta.GetAsDeltaGetResponseAsync((config) =>
{
config.QueryParameters.Top = (int)InitialMessageDownloadCountPerFolder;
config.QueryParameters.Select = outlookMessageSelectParameters;
config.QueryParameters.Orderby = ["receivedDateTime desc"];
}, cancellationToken).ConfigureAwait(false);
}
else
{
var currentDeltaToken = folder.DeltaToken;
_logger.Debug("Sync identifier found for Folder {FolderName}. Performing delta sync.", folder.FolderName);
_logger.Debug("Current delta token: {CurrentDeltaToken}", currentDeltaToken);
var requestInformation = _graphClient.Me.MailFolders[folder.RemoteFolderId].Messages.Delta.ToGetRequestInformation((config) =>
{
config.QueryParameters.Top = (int)InitialMessageDownloadCountPerFolder;
config.QueryParameters.Select = outlookMessageSelectParameters;
config.QueryParameters.Orderby = ["receivedDateTime desc"];
});
requestInformation.UrlTemplate = requestInformation.UrlTemplate.Insert(requestInformation.UrlTemplate.Length - 1, ",%24deltatoken");
requestInformation.QueryParameters.Add("%24deltatoken", currentDeltaToken);
messageCollectionPage = await _graphClient.RequestAdapter.SendAsync(requestInformation, Microsoft.Graph.Me.MailFolders.Item.Messages.Delta.DeltaGetResponse.CreateFromDiscriminatorValue);
}
var messageIteratorAsync = PageIterator<Message, Microsoft.Graph.Me.MailFolders.Item.Messages.Delta.DeltaGetResponse>.CreatePageIterator(_graphClient, messageCollectionPage, async (item) =>
{
try
{
await _handleItemRetrievalSemaphore.WaitAsync();
return await HandleItemRetrievedAsync(item, folder, downloadedMessageIds, cancellationToken);
}
catch (Exception ex)
{
_logger.Error(ex, "Error occurred while handling item {Id} for folder {FolderName}", item.Id, folder.FolderName);
}
finally
{
_handleItemRetrievalSemaphore.Release();
}
return true;
});
await messageIteratorAsync
.IterateAsync(cancellationToken)
.ConfigureAwait(false);
latestDeltaLink = messageIteratorAsync.Deltalink;
if (downloadedMessageIds.Any())
{
_logger.Debug("Downloaded {Count} messages for folder {FolderName}", downloadedMessageIds.Count, folder.FolderName);
}
_logger.Debug("Iterator completed for folder {FolderName}", folder.FolderName);
_logger.Debug("Extracted latest delta link is {LatestDeltaLink}", latestDeltaLink);
//Store delta link for tracking new changes.
if (!string.IsNullOrEmpty(latestDeltaLink))
{
// Parse Delta Token from Delta Link since v5 of Graph SDK works based on the token, not the link.
var deltaToken = GetDeltaTokenFromDeltaLink(latestDeltaLink);
await _outlookChangeProcessor.UpdateFolderDeltaSynchronizationIdentifierAsync(folder.Id, deltaToken).ConfigureAwait(false);
}
await _outlookChangeProcessor.UpdateFolderLastSyncDateAsync(folder.Id).ConfigureAwait(false);
return downloadedMessageIds;
}
private string GetDeltaTokenFromDeltaLink(string deltaLink)
=> Regex.Split(deltaLink, "deltatoken=")[1];
private bool IsResourceDeleted(IDictionary<string, object> additionalData)
=> additionalData != null && additionalData.ContainsKey("@removed");
private bool IsResourceUpdated(IDictionary<string, object> additionalData)
=> additionalData == null || !additionalData.Any();
private async Task<bool> HandleFolderRetrievedAsync(MailFolder folder, OutlookSpecialFolderIdInformation outlookSpecialFolderIdInformation, CancellationToken cancellationToken = default)
{
if (IsResourceDeleted(folder.AdditionalData))
{
await _outlookChangeProcessor.DeleteFolderAsync(Account.Id, folder.Id).ConfigureAwait(false);
}
else if (IsResourceUpdated(folder.AdditionalData))
{
// TODO
Debugger.Break();
}
else
{
// New folder created.
var item = folder.GetLocalFolder(Account.Id);
if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.InboxId))
item.SpecialFolderType = SpecialFolderType.Inbox;
else if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.SentId))
item.SpecialFolderType = SpecialFolderType.Sent;
else if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.DraftId))
item.SpecialFolderType = SpecialFolderType.Draft;
else if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.TrashId))
item.SpecialFolderType = SpecialFolderType.Deleted;
else if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.JunkId))
item.SpecialFolderType = SpecialFolderType.Junk;
else if (item.RemoteFolderId.Equals(outlookSpecialFolderIdInformation.ArchiveId))
item.SpecialFolderType = SpecialFolderType.Archive;
else
item.SpecialFolderType = SpecialFolderType.Other;
// Automatically mark special folders as Sticky for better visibility.
item.IsSticky = item.SpecialFolderType != SpecialFolderType.Other;
// By default, all non-others are system folder.
item.IsSystemFolder = item.SpecialFolderType != SpecialFolderType.Other;
// By default, all special folders update unread count in the UI except Trash.
item.ShowUnreadCount = item.SpecialFolderType != SpecialFolderType.Deleted || item.SpecialFolderType != SpecialFolderType.Other;
await _outlookChangeProcessor.InsertFolderAsync(item).ConfigureAwait(false);
}
return true;
}
private async Task<bool> HandleItemRetrievedAsync(Message item, MailItemFolder folder, IList<string> downloadedMessageIds, CancellationToken cancellationToken = default)
{
if (IsResourceDeleted(item.AdditionalData))
{
// Deleting item with this override instead of the other one that deletes all mail copies.
// Outlook mails have 1 assignment per-folder, unlike Gmail that has one to many.
await _outlookChangeProcessor.DeleteAssignmentAsync(Account.Id, item.Id, folder.RemoteFolderId).ConfigureAwait(false);
}
else if (IsResourceUpdated(item.AdditionalData))
{
// Some of the properties of the item are updated.
if (item.IsRead != null)
{
await _outlookChangeProcessor.ChangeMailReadStatusAsync(item.Id, item.IsRead.GetValueOrDefault()).ConfigureAwait(false);
}
if (item.Flag?.FlagStatus != null)
{
await _outlookChangeProcessor.ChangeFlagStatusAsync(item.Id, item.Flag.FlagStatus.GetValueOrDefault() == FollowupFlagStatus.Flagged)
.ConfigureAwait(false);
}
}
else
{
// Package may return null on some cases mapping the remote draft to existing local draft.
var newMailPackages = await CreateNewMailPackagesAsync(item, folder, cancellationToken);
if (newMailPackages != null)
{
foreach (var package in newMailPackages)
{
// Only add to downloaded message ids if it's inserted successfuly.
// Updates should not be added to the list because they are not new.
bool isInserted = await _outlookChangeProcessor.CreateMailAsync(Account.Id, package).ConfigureAwait(false);
if (isInserted)
{
downloadedMessageIds.Add(package.Copy.Id);
}
}
}
}
return true;
}
private async Task SynchronizeFoldersAsync(CancellationToken cancellationToken = default)
{
// Gather special folders by default.
// Others will be other type.
// Get well known folder ids by batch.
var wellKnownFolderIdBatch = new BatchRequestContentCollection(_graphClient);
var inboxRequest = _graphClient.Me.MailFolders[INBOX_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var sentRequest = _graphClient.Me.MailFolders[SENT_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var deletedRequest = _graphClient.Me.MailFolders[DELETED_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var junkRequest = _graphClient.Me.MailFolders[JUNK_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var draftsRequest = _graphClient.Me.MailFolders[DRAFTS_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var archiveRequest = _graphClient.Me.MailFolders[ARCHIVE_NAME].ToGetRequestInformation((t) => { t.QueryParameters.Select = ["id"]; });
var inboxId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(inboxRequest);
var sentId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(sentRequest);
var deletedId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(deletedRequest);
var junkId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(junkRequest);
var draftsId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(draftsRequest);
var archiveId = await wellKnownFolderIdBatch.AddBatchRequestStepAsync(archiveRequest);
var returnedResponse = await _graphClient.Batch.PostAsync(wellKnownFolderIdBatch, cancellationToken).ConfigureAwait(false);
var inboxFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(inboxId)).Id;
var sentFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(sentId)).Id;
var deletedFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(deletedId)).Id;
var junkFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(junkId)).Id;
var draftsFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(draftsId)).Id;
var archiveFolderId = (await returnedResponse.GetResponseByIdAsync<MailFolder>(archiveId)).Id;
var specialFolderInfo = new OutlookSpecialFolderIdInformation(inboxFolderId, deletedFolderId, junkFolderId, draftsFolderId, sentFolderId, archiveFolderId);
Microsoft.Graph.Me.MailFolders.Delta.DeltaGetResponse graphFolders = null;
if (string.IsNullOrEmpty(Account.SynchronizationDeltaIdentifier))
{
// Initial folder sync.
var deltaRequest = _graphClient.Me.MailFolders.Delta.ToGetRequestInformation();
deltaRequest.UrlTemplate = deltaRequest.UrlTemplate.Insert(deltaRequest.UrlTemplate.Length - 1, ",includehiddenfolders");
deltaRequest.QueryParameters.Add("includehiddenfolders", "true");
graphFolders = await _graphClient.RequestAdapter.SendAsync(deltaRequest,
Microsoft.Graph.Me.MailFolders.Delta.DeltaGetResponse.CreateFromDiscriminatorValue,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
var currentDeltaLink = Account.SynchronizationDeltaIdentifier;
var deltaRequest = _graphClient.Me.MailFolders.Delta.ToGetRequestInformation();
deltaRequest.UrlTemplate = deltaRequest.UrlTemplate.Insert(deltaRequest.UrlTemplate.Length - 1, ",%24deltaToken");
deltaRequest.QueryParameters.Add("%24deltaToken", currentDeltaLink);
graphFolders = await _graphClient.RequestAdapter.SendAsync(deltaRequest,
Microsoft.Graph.Me.MailFolders.Delta.DeltaGetResponse.CreateFromDiscriminatorValue,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
var iterator = PageIterator<MailFolder, Microsoft.Graph.Me.MailFolders.Delta.DeltaGetResponse>.CreatePageIterator(_graphClient, graphFolders, (folder) =>
{
return HandleFolderRetrievedAsync(folder, specialFolderInfo, cancellationToken);
});
await iterator.IterateAsync();
if (!string.IsNullOrEmpty(iterator.Deltalink))
{
// Get the second part of the query that its the deltaToken
var deltaToken = iterator.Deltalink.Split('=')[1];
var latestAccountDeltaToken = await _outlookChangeProcessor.UpdateAccountDeltaSynchronizationIdentifierAsync(Account.Id, deltaToken);
if (!string.IsNullOrEmpty(latestAccountDeltaToken))
{
Account.SynchronizationDeltaIdentifier = latestAccountDeltaToken;
}
}
}
#region Mail Integration
public override bool DelaySendOperationSynchronization() => true;
public override IEnumerable<IRequestBundle<RequestInformation>> Move(BatchMoveRequest request)
{
var requestBody = new Microsoft.Graph.Me.Messages.Item.Move.MovePostRequestBody()
{
DestinationId = request.ToFolder.RemoteFolderId
};
return CreateBatchedHttpBundle(request, (item) =>
{
return _graphClient.Me.Messages[item.Item.Id.ToString()].Move.ToPostRequestInformation(requestBody);
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> ChangeFlag(BatchChangeFlagRequest request)
{
return CreateBatchedHttpBundle(request, (item) =>
{
var message = new Message()
{
Flag = new FollowupFlag() { FlagStatus = request.IsFlagged ? FollowupFlagStatus.Flagged : FollowupFlagStatus.NotFlagged }
};
return _graphClient.Me.Messages[item.Item.Id.ToString()].ToPatchRequestInformation(message);
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> MarkRead(BatchMarkReadRequest request)
{
return CreateBatchedHttpBundle(request, (item) =>
{
var message = new Message()
{
IsRead = request.IsRead
};
return _graphClient.Me.Messages[item.Item.Id].ToPatchRequestInformation(message);
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> Delete(BatchDeleteRequest request)
{
return CreateBatchedHttpBundle(request, (item) =>
{
return _graphClient.Me.Messages[item.Item.Id].ToDeleteRequestInformation();
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> MoveToFocused(BatchMoveToFocusedRequest request)
{
return CreateBatchedHttpBundleFromGroup(request, (item) =>
{
if (item is MoveToFocusedRequest moveToFocusedRequest)
{
var message = new Message()
{
InferenceClassification = moveToFocusedRequest.MoveToFocused ? InferenceClassificationType.Focused : InferenceClassificationType.Other
};
return _graphClient.Me.Messages[moveToFocusedRequest.Item.Id].ToPatchRequestInformation(message);
}
throw new Exception("Invalid request type.");
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> AlwaysMoveTo(BatchAlwaysMoveToRequest request)
{
return CreateBatchedHttpBundle<Message>(request, (item) =>
{
if (item is AlwaysMoveToRequest alwaysMoveToRequest)
{
var inferenceClassificationOverride = new InferenceClassificationOverride
{
ClassifyAs = alwaysMoveToRequest.MoveToFocused ? InferenceClassificationType.Focused : InferenceClassificationType.Other,
SenderEmailAddress = new EmailAddress
{
Name = alwaysMoveToRequest.Item.FromName,
Address = alwaysMoveToRequest.Item.FromAddress
}
};
return _graphClient.Me.InferenceClassification.Overrides.ToPostRequestInformation(inferenceClassificationOverride);
}
throw new Exception("Invalid request type.");
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> CreateDraft(BatchCreateDraftRequest request)
{
return CreateHttpBundle<Message>(request, (item) =>
{
if (item is CreateDraftRequest createDraftRequest)
{
createDraftRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage.Prepare(EncodingConstraint.None);
var plainTextBytes = Encoding.UTF8.GetBytes(createDraftRequest.DraftPreperationRequest.CreatedLocalDraftMimeMessage.ToString());
var base64Encoded = Convert.ToBase64String(plainTextBytes);
var requestInformation = _graphClient.Me.Messages.ToPostRequestInformation(new Message());
requestInformation.Headers.Clear();// replace the json content header
requestInformation.Headers.Add("Content-Type", "text/plain");
requestInformation.SetStreamContent(new MemoryStream(Encoding.UTF8.GetBytes(base64Encoded)), "text/plain");
return requestInformation;
}
return default;
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> SendDraft(BatchSendDraftRequestRequest request)
{
var sendDraftPreparationRequest = request.Request;
// 1. Delete draft
// 2. Create new Message with new MIME.
// 3. Make sure that conversation id is tagged correctly for replies.
var mailCopyId = sendDraftPreparationRequest.MailItem.Id;
var mimeMessage = sendDraftPreparationRequest.Mime;
var batchDeleteRequest = new BatchDeleteRequest(new List<IRequest>()
{
new DeleteRequest(sendDraftPreparationRequest.MailItem)
});
var deleteBundle = Delete(batchDeleteRequest).ElementAt(0);
mimeMessage.Prepare(EncodingConstraint.None);
var plainTextBytes = Encoding.UTF8.GetBytes(mimeMessage.ToString());
var base64Encoded = Convert.ToBase64String(plainTextBytes);
var outlookMessage = new Message()
{
ConversationId = sendDraftPreparationRequest.MailItem.ThreadId
};
// Apply importance here as well just in case.
if (mimeMessage.Importance != MessageImportance.Normal)
outlookMessage.Importance = mimeMessage.Importance == MessageImportance.High ? Importance.High : Importance.Low;
var body = new Microsoft.Graph.Me.SendMail.SendMailPostRequestBody()
{
Message = outlookMessage
};
var sendRequest = _graphClient.Me.SendMail.ToPostRequestInformation(body);
sendRequest.Headers.Clear();
sendRequest.Headers.Add("Content-Type", "text/plain");
var stream = new MemoryStream(Encoding.UTF8.GetBytes(base64Encoded));
sendRequest.SetStreamContent(stream, "text/plain");
var sendMailRequest = new HttpRequestBundle<RequestInformation>(sendRequest, request);
return [deleteBundle, sendMailRequest];
}
public override IEnumerable<IRequestBundle<RequestInformation>> Archive(BatchArchiveRequest request)
=> Move(new BatchMoveRequest(request.Items, request.FromFolder, request.ToFolder));
public override async Task DownloadMissingMimeMessageAsync(IMailItem mailItem,
MailKit.ITransferProgress transferProgress = null,
CancellationToken cancellationToken = default)
{
var mimeMessage = await DownloadMimeMessageAsync(mailItem.Id, cancellationToken).ConfigureAwait(false);
await _outlookChangeProcessor.SaveMimeFileAsync(mailItem.FileId, mimeMessage, Account.Id).ConfigureAwait(false);
}
public override IEnumerable<IRequestBundle<RequestInformation>> RenameFolder(RenameFolderRequest request)
{
return CreateHttpBundleWithResponse<MailFolder>(request, (item) =>
{
if (item is not RenameFolderRequest renameFolderRequest)
throw new ArgumentException($"Renaming folder must be handled with '{nameof(RenameFolderRequest)}'");
var requestBody = new MailFolder
{
DisplayName = request.NewFolderName,
};
return _graphClient.Me.MailFolders[request.Folder.RemoteFolderId].ToPatchRequestInformation(requestBody);
});
}
public override IEnumerable<IRequestBundle<RequestInformation>> EmptyFolder(EmptyFolderRequest request)
=> Delete(new BatchDeleteRequest(request.MailsToDelete.Select(a => new DeleteRequest(a))));
public override IEnumerable<IRequestBundle<RequestInformation>> MarkFolderAsRead(MarkFolderAsReadRequest request)
=> MarkRead(new BatchMarkReadRequest(request.MailsToMarkRead.Select(a => new MarkReadRequest(a, true)), true));
#endregion
public override async Task ExecuteNativeRequestsAsync(IEnumerable<IRequestBundle<RequestInformation>> batchedRequests, CancellationToken cancellationToken = default)
{
var batchRequestInformations = BatchExtension.Batch(batchedRequests, (int)MaximumAllowedBatchRequestSize);
foreach (var batch in batchRequestInformations)
{
var batchContent = new BatchRequestContentCollection(_graphClient);
var itemCount = batch.Count();
for (int i = 0; i < itemCount; i++)
{
var bundle = batch.ElementAt(i);
var request = bundle.Request;
var nativeRequest = bundle.NativeRequest;
request.ApplyUIChanges();
await batchContent.AddBatchRequestStepAsync(nativeRequest).ConfigureAwait(false);
// Map BundleId to batch request step's key.
// This is how we can identify which step succeeded or failed in the bundle.
bundle.BundleId = batchContent.BatchRequestSteps.ElementAt(i).Key;
}
if (!batchContent.BatchRequestSteps.Any())
continue;
// Execute batch. This will collect responses from network call for each batch step.
var batchRequestResponse = await _graphClient.Batch.PostAsync(batchContent).ConfigureAwait(false);
// Check responses for each bundle id.
// Each bundle id must return some HttpResponseMessage ideally.
var bundleIds = batchContent.BatchRequestSteps.Select(a => a.Key);
// TODO: Handling responses. They used to work in v1 core, but not in v2.
foreach (var bundleId in bundleIds)
{
var bundle = batch.FirstOrDefault(a => a.BundleId == bundleId);
if (bundle == null)
continue;
var httpResponseMessage = await batchRequestResponse.GetResponseByIdAsync(bundleId);
using (httpResponseMessage)
{
await ProcessSingleNativeRequestResponseAsync(bundle, httpResponseMessage, cancellationToken).ConfigureAwait(false);
}
}
}
}
private async Task ProcessSingleNativeRequestResponseAsync(IRequestBundle<RequestInformation> bundle,
HttpResponseMessage httpResponseMessage,
CancellationToken cancellationToken = default)
{
if (!httpResponseMessage.IsSuccessStatusCode)
{
throw new SynchronizerException(string.Format(Translator.Exception_SynchronizerFailureHTTP, httpResponseMessage.StatusCode));
}
else if (bundle is HttpRequestBundle<RequestInformation, Message> messageBundle)
{
var outlookMessage = await messageBundle.DeserializeBundleAsync(httpResponseMessage, cancellationToken);
if (outlookMessage == null) return;
// TODO: Handle new message added or updated.
}
else if (bundle is HttpRequestBundle<RequestInformation, Microsoft.Graph.Models.MailFolder> folderBundle)
{
var outlookFolder = await folderBundle.DeserializeBundleAsync(httpResponseMessage, cancellationToken);
if (outlookFolder == null) return;
// TODO: Handle new folder added or updated.
}
else if (bundle is HttpRequestBundle<RequestInformation, MimeMessage> mimeBundle)
{
// TODO: Handle mime retrieve message.
}
}
private async Task<MimeMessage> DownloadMimeMessageAsync(string messageId, CancellationToken cancellationToken = default)
{
var mimeContentStream = await _graphClient.Me.Messages[messageId].Content.GetAsync(null, cancellationToken).ConfigureAwait(false);
return await MimeMessage.LoadAsync(mimeContentStream).ConfigureAwait(false);
}
public override async Task<List<NewMailItemPackage>> CreateNewMailPackagesAsync(Message message, MailItemFolder assignedFolder, CancellationToken cancellationToken = default)
{
bool isMailExists = await _outlookChangeProcessor.IsMailExistsAsync(message.Id);
if (isMailExists)
{
return null;
}
var mimeMessage = await DownloadMimeMessageAsync(message.Id, cancellationToken).ConfigureAwait(false);
var mailCopy = message.AsMailCopy();
if (message.IsDraft.GetValueOrDefault()
&& mimeMessage.Headers.Contains(Wino.Domain.Constants.WinoLocalDraftHeader)
&& Guid.TryParse(mimeMessage.Headers[Wino.Domain.Constants.WinoLocalDraftHeader], out Guid localDraftCopyUniqueId))
{
// This message belongs to existing local draft copy.
// We don't need to create a new mail copy for this message, just update the existing one.
bool isMappingSuccessful = await _outlookChangeProcessor.MapLocalDraftAsync(Account.Id, localDraftCopyUniqueId, mailCopy.Id, mailCopy.DraftId, mailCopy.ThreadId);
if (isMappingSuccessful) return null;
// Local copy doesn't exists. Continue execution to insert mail copy.
}
// Outlook messages can only be assigned to 1 folder at a time.
// Therefore we don't need to create multiple copies of the same message for different folders.
var package = new NewMailItemPackage(mailCopy, mimeMessage, assignedFolder.RemoteFolderId);
return [package];
}
}
}

View File

@@ -0,0 +1,36 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>disable</Nullable>
<RootNamespace>Wino.Synchronization</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommunityToolkit.Diagnostics" Version="8.2.2" />
<PackageReference Include="CommunityToolkit.Mvvm" Version="8.2.2" />
<PackageReference Include="Google.Apis.Gmail.v1" Version="1.68.0.3427" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
<PackageReference Include="HtmlKit" Version="1.1.0" />
<PackageReference Include="MailKit" Version="4.7.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Graph" Version="5.56.0" />
<PackageReference Include="MimeKit" Version="4.7.1" />
<PackageReference Include="morelinq" Version="4.3.0" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="5.1.2" />
<PackageReference Include="Serilog" Version="4.0.0" />
<PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
<PackageReference Include="Serilog.Sinks.Debug" Version="3.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Wino.Core.Domain\Wino.Domain.csproj" />
<ProjectReference Include="..\Wino.Messaging\Wino.Messaging.csproj" />
<ProjectReference Include="..\Wino.Services\Wino.Services.csproj" />
</ItemGroup>
</Project>