// Spludlow Software // Copyright © Samuel P. Ludlow 2020 All Rights Reserved // Distributed under the terms of the GNU General Public License version 3 // Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE // https://www.spludlow.co.uk/LICENCE.TXT // The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission // v1.14 using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Data; using System.Text.RegularExpressions; using System.Reflection; namespace Spludlow.Email { public class ReceiveMail { public static int CountDrop() { string dropDirectory = Spludlow.Config.Get("SpludlowMail.Drop"); string[] filenames = Directory.GetFiles(dropDirectory, "*.eml"); return filenames.Length; } public static void StartReadDrop(string host) { string dropDirectory = Spludlow.Config.Get("SpludlowMail.Drop"); Spludlow.Call.Queue(host + ":Run", "Spludlow", "Spludlow.Email.ReceiveMail", "ReadDrop"); } public static void ReadDrop() { string dropDirectory = Spludlow.Config.Get("SpludlowMail.Drop"); ImportDirectory(dropDirectory, true); } public static void ImportDirectory(string directory, bool deleteFile) { string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); Spludlow.Data.DAL.CommandInfo insertMailMessages = database.MakeInserter("MailMessages"); Spludlow.Data.DAL.CommandInfo insertMailMesssageAddresses = database.MakeInserter("MailMesssageAddresses"); Spludlow.Data.DAL.CommandInfo insertMailAddresses = database.MakeInserter("MailAddresses"); List messages = new List(); List filenames = new List(); foreach (string filename in Directory.GetFiles(directory)) { string exention = Path.GetExtension(filename).ToLower(); if (exention != ".eml") continue; Spludlow.Email.SimpleMailMessage message = new SimpleMailMessage(filename); messages.Add(message); filenames.Add(filename); } Spludlow.Schema.MailMessagesDataTable messagesTable = new Schema.MailMessagesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(messagesTable); messagesTable.Constraints.Clear(); Spludlow.Schema.MailMesssageAddressesDataTable resolverTable = new Schema.MailMesssageAddressesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(resolverTable); resolverTable.Constraints.Clear(); Spludlow.Schema.MailAddressesDataTable addressTable = new Schema.MailAddressesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(addressTable); addressTable.Constraints.Clear(); messagesTable.Columns.Add("Filename", typeof(string)); for (int messageIndex = 0; messageIndex < messages.Count; ++messageIndex) { Spludlow.Email.SimpleMailMessage message = messages[messageIndex]; string filename = filenames[messageIndex]; string extention = Path.GetExtension(filename).ToLower(); Spludlow.Schema.MailMessagesRow messageRow = messagesTable.NewMailMessagesRow(); messageRow.FileExtention = extention; messageRow.FileSize = (int)Spludlow.Io.Files.FileLength(filename); messageRow.FileLastWriteTime = File.GetLastWriteTime(filename); messageRow.MailUser = ""; messageRow.MailFolder = ""; messageRow.Sender = message.FromAddress[0][0]; messageRow.Receiver = message.XReceiver; messageRow.ReplyTo = message.ReplyToAddress[0][0]; messageRow.ReturnPath = ""; if (message.ReturnPaths.Length > 0) messageRow.ReturnPath = message.ReturnPaths[0][0]; messageRow.Subject = message.Subject; messageRow.TextBodyLength = message.TextBody.Length; messageRow.HTMLBodyLength = message.HTMLBody.Length; messageRow.ReceivedTime = message.ReceivedTime; messageRow.SentOn = message.SentOn; messageRow.AttachmentCount = -1; messageRow.AttachmentsSize = -1; messageRow["Filename"] = filename; messagesTable.Rows.Add(messageRow); string[][][] addressesGroup = new string[][][] { message.BCCAddress, message.CCAddress, message.ToAddress, message.FromAddress, message.ReplyToAddress, message.ReturnPaths }; string[] addressTypes = new string[] { "BCC", "CC", "TO", "FROM", "REP", "RET" }; for (int index = 0; index < addressTypes.Length; ++index) { string[][] addresses = addressesGroup[index]; string addressType = addressTypes[index]; foreach (string[] address in addresses) { Spludlow.Schema.MailAddressesRow addressRow = addressTable.NewMailAddressesRow(); addressRow.Address = address[0]; addressRow.Display = address[1]; addressTable.Rows.Add(addressRow); Spludlow.Schema.MailMesssageAddressesRow resolverRow = resolverTable.NewMailMesssageAddressesRow(); resolverRow.MailMessageId = messageRow.MailMessageId; resolverRow.MailAddressId = addressRow.MailAddressId; resolverRow.MailAddressType = addressType; resolverTable.Rows.Add(resolverRow); } } } addressTable.PrimaryKey = new DataColumn[] { addressTable.MailAddressIdColumn }; foreach (Spludlow.Schema.MailMessagesRow messageRow in messagesTable.Rows) { string filename = (string)messageRow["Filename"]; string tempFilename = Spludlow.Io.FileStore.TempName(storeDirectory, messageRow.FileExtention); File.Copy(filename, tempFilename); database.Begin(); try { DataRow[] resolverRows = resolverTable.Select("MailMessageId = " + messageRow.MailMessageId); messageRow.MailMessageId = database.Insert(insertMailMessages, messageRow); foreach (Spludlow.Schema.MailMesssageAddressesRow resolverRow in resolverRows) { Spludlow.Schema.MailAddressesRow addressRow = addressTable.FindByMailAddressId(resolverRow.MailAddressId); if (addressRow == null) throw new ApplicationException("Can not find Address Row"); addressRow.MailAddressId = database.Insert(insertMailAddresses, addressRow); resolverRow.MailMessageId = messageRow.MailMessageId; resolverRow.MailAddressId = addressRow.MailAddressId; database.Insert(insertMailMesssageAddresses, resolverRow); } string storeFilename = Spludlow.Io.FileStore.FilePath(messageRow.MailMessageId, storeDirectory, messageRow.FileExtention, true); File.Move(tempFilename, storeFilename); database.Commit(); if (deleteFile == true) File.Delete(filename); } catch { database.Rollback(); throw; } } Spludlow.Log.Report("ReceiveMail", messagesTable, addressTable, resolverTable); } // Duplicate Code Above and below !!!!!!!!!!!!!!!!! public static void ImportFile(string filename, bool deleteFile) { Spludlow.Schema.MailMessagesDataTable messagesTable = new Schema.MailMessagesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(messagesTable); messagesTable.Constraints.Clear(); Spludlow.Schema.MailMesssageAddressesDataTable resolverTable = new Schema.MailMesssageAddressesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(resolverTable); resolverTable.Constraints.Clear(); Spludlow.Schema.MailAddressesDataTable addressTable = new Schema.MailAddressesDataTable(); Spludlow.Data.ADO.RemoveKeyConstraints(addressTable); addressTable.Constraints.Clear(); Spludlow.Email.SimpleMailMessage message = new SimpleMailMessage(filename); string extention = Path.GetExtension(filename).ToLower(); Spludlow.Schema.MailMessagesRow messageRow = messagesTable.NewMailMessagesRow(); messageRow.FileExtention = extention; messageRow.FileSize = (int)Spludlow.Io.Files.FileLength(filename); messageRow.FileLastWriteTime = File.GetLastWriteTime(filename); messageRow.MailUser = ""; messageRow.MailFolder = ""; messageRow.Sender = message.FromAddress[0][0]; messageRow.Receiver = message.XReceiver; messageRow.ReplyTo = message.ReplyToAddress[0][0]; messageRow.ReturnPath = ""; if (message.ReturnPaths.Length > 0) messageRow.ReturnPath = message.ReturnPaths[0][0]; messageRow.Subject = message.Subject; messageRow.TextBodyLength = message.TextBody.Length; messageRow.HTMLBodyLength = message.HTMLBody.Length; messageRow.ReceivedTime = message.ReceivedTime; messageRow.SentOn = message.SentOn; messageRow.AttachmentCount = -1; messageRow.AttachmentsSize = -1; //messageRow["Filename"] = filename; messagesTable.Rows.Add(messageRow); string[][][] addressesGroup = new string[][][] { message.BCCAddress, message.CCAddress, message.ToAddress, message.FromAddress, message.ReplyToAddress, message.ReturnPaths }; string[] addressTypes = new string[] { "BCC", "CC", "TO", "FROM", "REP", "RET" }; for (int index = 0; index < addressTypes.Length; ++index) { string[][] addresses = addressesGroup[index]; string addressType = addressTypes[index]; foreach (string[] address in addresses) { Spludlow.Schema.MailAddressesRow addressRow = addressTable.NewMailAddressesRow(); addressRow.Address = address[0]; addressRow.Display = address[1]; addressTable.Rows.Add(addressRow); Spludlow.Schema.MailMesssageAddressesRow resolverRow = resolverTable.NewMailMesssageAddressesRow(); resolverRow.MailMessageId = messageRow.MailMessageId; resolverRow.MailAddressId = addressRow.MailAddressId; resolverRow.MailAddressType = addressType; resolverTable.Rows.Add(resolverRow); } } addressTable.PrimaryKey = new DataColumn[] { addressTable.MailAddressIdColumn }; string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); Spludlow.Data.DAL.CommandInfo insertMailMessages = database.MakeInserter("MailMessages"); Spludlow.Data.DAL.CommandInfo insertMailMesssageAddresses = database.MakeInserter("MailMesssageAddresses"); Spludlow.Data.DAL.CommandInfo insertMailAddresses = database.MakeInserter("MailAddresses"); string tempFilename = Spludlow.Io.FileStore.TempName(storeDirectory, messageRow.FileExtention); File.Copy(filename, tempFilename); database.Begin(); try { DataRow[] resolverRows = resolverTable.Select("MailMessageId = " + messageRow.MailMessageId); messageRow.MailMessageId = database.Insert(insertMailMessages, messageRow); foreach (Spludlow.Schema.MailMesssageAddressesRow resolverRow in resolverRows) { Spludlow.Schema.MailAddressesRow addressRow = addressTable.FindByMailAddressId(resolverRow.MailAddressId); if (addressRow == null) throw new ApplicationException("Can not find Address Row"); addressRow.MailAddressId = database.Insert(insertMailAddresses, addressRow); resolverRow.MailMessageId = messageRow.MailMessageId; resolverRow.MailAddressId = addressRow.MailAddressId; database.Insert(insertMailMesssageAddresses, resolverRow); } string storeFilename = Spludlow.Io.FileStore.FilePath(messageRow.MailMessageId, storeDirectory, messageRow.FileExtention, true); File.Move(tempFilename, storeFilename); database.Commit(); if (deleteFile == true) File.Delete(filename); } catch { database.Rollback(); throw; } } public static void ForwardEmail(string filename, string emailAddress, string pickupDirectory, bool deleteSourceFile) { byte[] newLine = new byte[] { 0xD, 0xA }; byte[] doubleNewLine = new byte[] { 0xD, 0xA, 0xD, 0xA }; HashSet keepHeaders = new HashSet(new string[] { "received:", "date:", "from:", "to:", "cc:", "bcc:", "subject:", "mime-version:", "content-type:", "return-path:", "message-id:", }); string targetFilename = pickupDirectory + @"\" + Path.GetFileName(filename); using (FileStream writeStream = new FileStream(targetFilename, FileMode.Create, FileAccess.Write, FileShare.None)) { byte[] writeData = Encoding.ASCII.GetBytes("To: " + emailAddress); writeStream.Write(writeData, 0, writeData.Length); writeStream.Write(newLine, 0, newLine.Length); using (FileStream readStream = new FileStream(filename, FileMode.Open)) { long bodyOffset = Spludlow.Io.BinaryIO.FindBinaryPattern(readStream, doubleNewLine); readStream.Position = 0; using (StreamReader reader = new StreamReader(readStream, Encoding.ASCII)) { bool inWanted = true; string line; while ((line = reader.ReadLine()) != null) { if (line.Length == 0) break; bool continuationLine = (line[0] == ' ' || line[0] == '\t'); if (continuationLine == false) { string headerName = line.ToLower(); int index = headerName.IndexOf(":"); if (index != -1) headerName = headerName.Substring(0, index + 1); if (keepHeaders.Contains(headerName) == false) { inWanted = false; continue; } inWanted = true; switch (headerName) { case "to:": line = "X-original-to:" + line.Substring(headerName.Length); break; case "cc:": line = "X-original-cc:" + line.Substring(headerName.Length); break; case "bcc:": line = "X-original-bcc:" + line.Substring(headerName.Length); break; } } else { if (inWanted == false) continue; } writeData = Encoding.ASCII.GetBytes(line); writeStream.Write(writeData, 0, writeData.Length); writeStream.Write(newLine, 0, newLine.Length); } if (bodyOffset != -1) { readStream.Position = bodyOffset + 2; readStream.CopyTo(writeStream); } } } } if (deleteSourceFile == true) File.Delete(filename); } private static string[][] FilenameBatches(string directoryName, int batchSize) { DataTable table = Spludlow.Data.TextTable.ReadText(new string[] { "FullName LastAccessTime", "String DateTime", }); DirectoryInfo directoryInfo = new DirectoryInfo(directoryName); foreach (FileInfo fileInfo in directoryInfo.GetFiles("*.eml")) { table.Rows.Add(new object[] { fileInfo.FullName, fileInfo.LastAccessTime }); } DataView view = new DataView(table); view.Sort = "LastAccessTime"; List batches = new List(); List current = new List(); foreach (DataRowView rowView in view) { DataRow row = rowView.Row; current.Add((string)row["FullName"]); if (current.Count == batchSize) { batches.Add(current.ToArray()); current = new List(); } } if (current.Count > 0) batches.Add(current.ToArray()); return batches.ToArray(); } public static void CleanStore() { Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create("@Spludlow"); DataSet dataSet = new DataSet(); dal.Fill(dataSet, "SELECT MailMessageId FROM MailMessages"); List ids = new List(); foreach (DataRow row in dataSet.Tables[0].Rows) ids.Add((uint)((long)row["MailMessageId"])); string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Io.FileStore.CleanStore(ids.ToArray(), storeDirectory, ".eml"); } public static DataSet QueryStore(int pageIndex, int pageSize) { Spludlow.Schema.MailMessagesDataTable table = new Schema.MailMessagesDataTable(); Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create("@Spludlow"); string commandText = dal.PageCommandText("SELECT * FROM MailMessages ORDER BY ReceivedTime DESC", pageIndex, pageSize); // WHERE (UserName = '') dal.Fill(table, commandText); return Spludlow.Data.ADO.WireDataSet(table); } public static DataSet QueryStore(int pageIndex, int pageSize, string folder, string order) { Spludlow.Schema.MailMessagesDataTable table = new Schema.MailMessagesDataTable(); Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create("@Spludlow"); string commandText = dal.PageCommandText("SELECT * FROM MailMessages @WHERE ORDER BY @ORDER", pageIndex, pageSize); commandText = commandText.Replace("@ORDER", order); string where = ""; if (folder != "") where = "WHERE ((MailFolder = '') OR (MailFolder = @MailFolder))"; commandText = commandText.Replace("@WHERE", where); if (where == "") dal.Fill(table, commandText); else dal.Fill(table, commandText, folder); return Spludlow.Data.ADO.WireDataSet(table); } public static DataTable AnalyiseMailHeaders() { string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create("@Spludlow"); DataTable table = dal.Select("SELECT MailMessageId, StoreExtension FROM MailMessages"); DataTable resultTable = Spludlow.Data.TextTable.ReadText(new string[] { "Name", "String", }); Spludlow.Data.ADO.ReflectNewTable(resultTable, typeof(Spludlow.Email.SimpleMailMessage)); Regex cleanWhite = new Regex(@"\s+"); foreach (DataRow row in table.Rows) { long id = (long)row["MailMessageId"]; string extention = (string)row["StoreExtension"]; string filename = Spludlow.Io.FileStore.FilePath(id, storeDirectory, extention, true); string name = Path.GetFileName(filename); Spludlow.Email.SimpleMailMessage message = new SimpleMailMessage(filename, null); DataRow resultRow = Spludlow.Data.ADO.ReflectNewRow(resultTable, message); resultRow["Name"] = name; } Spludlow.Log.Report("AnalyiseMailHeaders", resultTable); return resultTable; } public static string[] MailFolders() { Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create("@Spludlow"); DataTable table = dal.Select("SELECT MailMessages.MailFolder, Count(1) AS FolderCount FROM MailMessages GROUP BY MailMessages.MailFolder ORDER BY FolderCount DESC"); List folders = new List(); foreach (DataRow row in table.Rows) folders.Add((string)row["MailFolder"]); return folders.ToArray(); } public static DataTable MessageAddresses(long mailMessageId) { Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); return database.Select("SELECT MailMesssageAddresses.*, MailAddresses.* " + "FROM MailAddresses INNER JOIN MailMesssageAddresses ON MailAddresses.MailAddressId = MailMesssageAddresses.MailAddressId WHERE MailMesssageAddresses.MailMessageId = " + mailMessageId); } /// /// Inserts are performed simpily and will create duplicate address records. /// This can be ran to remove the duplicates and update the resolver references /// TODO: Handle display name changes, think it just needs to work descending so it gets latest /// public static void CompressAddressResolvers() { Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); Spludlow.Schema.MailAddressesDataTable addressTable = new Schema.MailAddressesDataTable(); Spludlow.Schema.MailMesssageAddressesDataTable resolverTable = new Schema.MailMesssageAddressesDataTable(); database.Fill(addressTable, "SELECT * FROM MailAddresses"); database.Fill(resolverTable, "SELECT * FROM MailMesssageAddresses"); Dictionary addressLookup = new Dictionary(); HashSet removeAddressIds = new HashSet(); foreach (Spludlow.Schema.MailAddressesRow row in addressTable.Rows) { string address = (string)row["Address"]; // leave case for now if (addressLookup.ContainsKey(address) == false) addressLookup.Add(address, row.MailAddressId); else removeAddressIds.Add(row.MailAddressId); } int resolverUpdateCount = 0; int resolverDeleteDupCount = 0; foreach (Spludlow.Schema.MailMesssageAddressesRow resolverRow in resolverTable.Rows) { Spludlow.Schema.MailAddressesRow addressRow = addressTable.FindByMailAddressId(resolverRow.MailAddressId); long newMailAddressId = addressLookup[addressRow.Address]; // Leave record alone already pointing to compressed if (resolverRow.MailAddressId == newMailAddressId) continue; // If creating a duplicate, seen messages with multiple same to addresses for example Spludlow.Schema.MailMesssageAddressesRow dupResolverRow = resolverTable.FindByMailMessageIdMailAddressIdMailAddressType(resolverRow.MailMessageId, newMailAddressId, resolverRow.MailAddressType); if (dupResolverRow != null) { resolverRow.Delete(); ++resolverDeleteDupCount; continue; } resolverRow.MailAddressId = newMailAddressId; ++resolverUpdateCount; } resolverTable.AcceptChanges(); Spludlow.Data.DAL.CommandInfo insertInfo = database.MakeInserter("MailMesssageAddresses"); database.Begin(); try { database.ExecuteNonQuery("DELETE FROM MailMesssageAddresses"); //foreach (Spludlow.Schema.MailMesssageAddressesRow resolverRow in resolverTable.Rows) // SLOW // database.Insert(insertInfo, resolverRow); foreach (long removeAddressId in removeAddressIds) database.ExecuteNonQuery("DELETE FROM MailAddresses WHERE MailAddressId = " + removeAddressId); database.Commit(); } catch { database.Rollback(); throw; } database.BulkInsert(resolverTable, "MailMesssageAddresses", null); // Can't do in transaction Spludlow.Log.Report("CompressAddressResolvers; addressTable rows:" + addressTable.Rows.Count + ", resolverTable rows: " + resolverTable.Rows.Count + ", resolverUpdateCount:" + resolverUpdateCount + ", resolverDeleteDupCount:" + resolverDeleteDupCount + ", removeAddressIds:" + removeAddressIds.Count); } public static void DumpAttachments(string outputDirectory, string tempDirectory) { string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); DataTable table = database.Select("SELECT MailMessageId, FileExtention FROM MailMessages"); using (Spludlow.TempDirectory tempDir = new TempDirectory(tempDirectory)) { int index = 0; foreach (DataRow row in table.Rows) { long mailMessageId = (long)row["MailMessageId"]; string extention = (string)row["FileExtention"]; string storeFilename = Spludlow.Io.FileStore.FilePath(mailMessageId, storeDirectory, extention, false); Spludlow.Email.SimpleMailMessage message = new Spludlow.Email.SimpleMailMessage(storeFilename, tempDir.Path); foreach (string sourceFilename in Directory.GetFiles(tempDir.Path)) { if (Path.GetFileName(sourceFilename).StartsWith("_") == false) { string targetFilename = outputDirectory + @"\" + index.ToString() + Path.GetExtension(sourceFilename); File.Copy(sourceFilename, targetFilename); ++index; } File.Delete(sourceFilename); } } } } public static void DeleteMailMessage(long mailMessageId, bool hardDelete) { string storeDirectory = Spludlow.Config.Get("SpludlowMail.Store"); Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("@Spludlow"); DataTable table = database.Select("SELECT MailMessageId, FileExtention FROM MailMessages WHERE MailMessageId = " + mailMessageId); if (table.Rows.Count == 0) throw new ApplicationException("DeleteMailMessage Delete not found: " + mailMessageId); if (hardDelete == false) { database.Update("MailMessages", new string[] { "DateDelete" }, mailMessageId, DateTime.Now); return; } string extention = (string)table.Rows[0]["FileExtention"]; string storeFilename = Spludlow.Io.FileStore.FilePath(mailMessageId, storeDirectory, extention, false); database.Begin(); try { database.ExecuteNonQuery("DELETE FROM MailMesssageAddresses WHERE MailMessageId = " + mailMessageId); database.ExecuteNonQuery("DELETE FROM MailMessages WHERE MailMessageId = " + mailMessageId); if (File.Exists(storeFilename) == true) File.Delete(storeFilename); database.Commit(); } catch { database.Rollback(); throw; } } } }