// Spludlow Software // Copyright © Samuel P. Ludlow 2020 All Rights Reserved // Distributed under the terms of the GNU General Public License version 3 // Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE // https://www.spludlow.co.uk/LICENCE.TXT // The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission // v1.14 using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Data; namespace Spludlow.MetSat { public class Receive { public static string _ReadyDirectory; public static string _ImportSourceId; public static Spludlow.Data.IDAL _Database; /// /// Identify and Receive files from EUMETCast received directory into the "Ready" /// static Receive() { _ReadyDirectory = Spludlow.Config.Get("MetSat.Ready"); _ImportSourceId = "EC"; _Database = Spludlow.Data.DAL.Create("@MetSat"); } /// /// Identify and move files from EUMETCast received directory into the "Ready" product directories /// Directory listing can be performed seperatly to increase speed in development when a lot of files are in EUMETCast received /// public static void ReceiveDirectory() { string dropHost = null; // If source host is running Spludlow this can be set to speed up directory listing Schema.ImportSourcesRow importSourceRow = ImportSourcesRow(_ImportSourceId); using (Spludlow.TempDirectory tempDir = new TempDirectory()) { string cacheInfoDirectory = tempDir + @"\DataSet"; SaveDirectory(dropHost, importSourceRow.Path, cacheInfoDirectory); ReceiveDirectory(cacheInfoDirectory, _ReadyDirectory, true); } } public static Schema.ImportSourcesRow ImportSourcesRow(string importSourceId) { Schema.ImportSourcesDataTable importSourcesDataTable = new Schema.ImportSourcesDataTable(); _Database.Fill(importSourcesDataTable, "SELECT * FROM ImportSources WHERE (ImportSourceId = @ImportSourceId)", importSourceId); if (importSourcesDataTable.Rows.Count != 1) throw new ApplicationException("MetSat Receive, Can not find importSource Row: " + importSourceId); Schema.ImportSourcesRow importSourceRow = (Schema.ImportSourcesRow)importSourcesDataTable.Rows[0]; return importSourceRow; } public static int SaveDirectory(string host, string directory, string cacheInfoDirectory) { if (Directory.Exists(cacheInfoDirectory) == true) Directory.Delete(cacheInfoDirectory, true); DataSet dirDataSet; if (host == null) dirDataSet = Spludlow.Io.DirectoryList.List(directory, true, true); else dirDataSet = Spludlow.Io.DirectoryList.List(host, directory, true, true); Spludlow.Data.TextTable.WriteDirectory(cacheInfoDirectory, dirDataSet); return dirDataSet.Tables[0].Rows.Count; } public static void ReadDirectory(string cacheInfoDirectory) { ReceiveDirectory(cacheInfoDirectory, null, false); } public static void ReceiveDirectory(string cacheInfoDirectory, string readyDirectory, bool deleteSource) { DataSet dataSet; dataSet = Spludlow.Data.TextTable.ReadDirectory(cacheInfoDirectory); Spludlow.SysTables.FileSystemInfosDataTable fileSystemTable = new SysTables.FileSystemInfosDataTable(); foreach (DataRow row in dataSet.Tables[0].Rows) fileSystemTable.ImportRow(row); fileSystemTable.Columns.Add("ProductId", typeof(string)); DataTable channelTable = Spludlow.Data.TextTable.ReadText("Channels", new string[] { "ChannelDirectory ChannelName DirectoryId FileCount FileTotalSize FileTotalSizeText FileMinDate FileMaxDate SubDirCount TakeCount LeaveCount ErrorCount Products", "String String Int32 Int32 Int64 String DateTime DateTime Int32 Int32 Int32 Int32 String", }); foreach (Spludlow.SysTables.FileSystemInfosRow channelDirectoryRow in fileSystemTable.Select("Type = 'D' AND Depth = 1")) { string channelDirectory = channelDirectoryRow.Path; string channelName = Path.GetFileName(channelDirectory); DataRow channelRow = channelTable.Rows.Add(channelDirectory, channelName, channelDirectoryRow.DirectoryId); DataRow[] fileRows = fileSystemTable.Select("Type = 'F' AND ParentDirectoryId = " + channelDirectoryRow.DirectoryId); DateTime minDate = DateTime.MaxValue; DateTime maxDate = DateTime.MinValue; long totalFileSize = 0; Dictionary products = new Dictionary(); foreach (Spludlow.SysTables.FileSystemInfosRow fileRow in fileRows) { if (fileRow.LastWriteTime < minDate) minDate = fileRow.LastWriteTime; if (fileRow.LastWriteTime > maxDate) maxDate = fileRow.LastWriteTime; totalFileSize += (long)fileRow.Length; string name = Path.GetFileName(fileRow.Path); string product = Spludlow.MetSat.ProductIdentify.DetermineProduct(name); fileRow["ProductId"] = product; if (products.ContainsKey(product) == false) products.Add(product, 0); ++products[product]; } StringBuilder productText = new StringBuilder(); foreach (string product in products.Keys) { if (productText.Length > 0) productText.Append(", "); productText.Append(product + ":" + products[product]); } channelRow["Products"] = productText.ToString(); if (fileRows.Length > 0) { channelRow["FileCount"] = fileRows.Length; channelRow["FileTotalSize"] = totalFileSize; channelRow["FileTotalSizeText"] = Spludlow.Text.DataSize(totalFileSize); channelRow["FileMinDate"] = minDate; channelRow["FileMaxDate"] = maxDate; } DataRow[] directoryRows = fileSystemTable.Select("Type = 'D' AND ParentDirectoryId = " + channelDirectoryRow.DirectoryId); channelRow["SubDirCount"] = directoryRows.Length; } if (readyDirectory != null) { foreach (DataRow channelRow in channelTable.Rows) { string channelDirectory = (string)channelRow["ChannelDirectory"]; string channelName = (string)channelRow["ChannelName"]; int directoryId = (int)channelRow["DirectoryId"]; DataRow[] fileRows = fileSystemTable.Select("Type = 'F' AND ParentDirectoryId = " + directoryId); int takeCount = 0; int leaveCount = 0; int errorCount = 0; HashSet checkedProductDirs = new HashSet(); using (Spludlow.TempDirectory tempDir = new TempDirectory()) { string arcDirectory = tempDir.Path + @"\ARC"; Directory.CreateDirectory(arcDirectory); foreach (Spludlow.SysTables.FileSystemInfosRow fileRow in fileRows) { string productId = (string)fileRow["ProductId"]; if (productId == "") { ++leaveCount; continue; } if (checkedProductDirs.Contains(productId) == false) { string targetDirectory = readyDirectory + @"\" + productId; if (Directory.Exists(targetDirectory) == false) Directory.CreateDirectory(targetDirectory); checkedProductDirs.Add(productId); } string extractFilename = ""; try { extractFilename = ExtractFile(fileRow.Path, tempDir.Path, arcDirectory); string sourceFilename = (extractFilename != null ? extractFilename : fileRow.Path); string name = Path.GetFileName(sourceFilename); StringBuilder filename = new StringBuilder(); filename.Append(readyDirectory); filename.Append(@"\"); filename.Append(productId); filename.Append(@"\"); filename.Append(name); string targetFilename = filename.ToString(); if (File.Exists(targetFilename) == true) { File.Delete(targetFilename); Spludlow.Log.Warning("MetSat Receive, ReceiveDirectory; Target file already exists, will replace, source:" + sourceFilename + " target:" + targetFilename); } File.Copy(sourceFilename, targetFilename); File.SetLastWriteTime(targetFilename, fileRow.LastWriteTime); if (deleteSource == true) File.Delete(fileRow.Path); ++takeCount; } catch (Exception ee) { Spludlow.Log.Error("MetSat Receive, ReceiveDirectory; Source File: " + fileRow.Path, ee); ++errorCount; } if (extractFilename != null) { Directory.Delete(arcDirectory, true); Directory.CreateDirectory(arcDirectory); } } } channelRow["TakeCount"] = takeCount; channelRow["LeaveCount"] = leaveCount; channelRow["ErrorCount"] = errorCount; } } dataSet = new DataSet(); dataSet.Tables.Add(channelTable); Spludlow.Log.Report("MetSat Receive, ReceiveDirectory: " + cacheInfoDirectory, dataSet); } private static string ExtractFile(string filename, string tempRootDirectory, string arcDirectory) { string name = Path.GetFileName(filename); if ((name.StartsWith("L-") == true || name.StartsWith("H-")) && name[name.Length - 2] == 'C') { if (filename.Contains(@":\") == true) return Spludlow.MetSat.Xrit2Pic.XRITDecompress(filename, arcDirectory, true); string tempFilename = tempRootDirectory + @"\" + name; File.Copy(filename, tempFilename); try { return Spludlow.MetSat.Xrit2Pic.XRITDecompress(tempFilename, arcDirectory, true); } finally { File.Delete(tempFilename); } } string extention = Path.GetExtension(filename).ToLower(); if (extention == ".bz2" || extention == ".gz") { Spludlow.Archive.Extract(filename, arcDirectory); string[] results = Directory.GetFiles(arcDirectory); if (results.Length == 1) return results[0]; if (results.Length > 1) throw new ApplicationException("ReceiveFile Extract, More than one file: " + filename); results = Directory.GetFiles(arcDirectory, "*", SearchOption.AllDirectories); if (results.Length != 1) throw new ApplicationException("ReceiveFile Extract, Not one deep file: " + filename + ", count: " + results.Length); return results[0]; } return null; } /// /// Delete old files based of LastWriteTime from the EUMETCast received directory /// Only un-identified product files will build up in the received directory, identified product files will have been moved to "Ready" /// public static void ClearReceivedDirectory(int daysHistory) { DateTime beforeDate = DateTime.Now.Date.AddDays(-daysHistory); DataTable table = Spludlow.Data.TextTable.ReadText(new string[] { "ChannelDirectory Removed Remaining Total", "String Int32 Int32 Int32", }); Schema.ImportSourcesRow importSourceRow = ImportSourcesRow(_ImportSourceId); if (importSourceRow.ImportSourceTypeId != "WD") throw new ApplicationException("MetSat ClearImportDirectory; Not a WD Import Source: " + importSourceRow.ImportSourceId); int totalRemoved = 0; foreach (string channelDirectory in Directory.GetDirectories(importSourceRow.Path)) { int remove = 0; int remain = 0; DirectoryInfo dirInfo = new DirectoryInfo(channelDirectory); foreach (FileInfo fileInfo in dirInfo.GetFiles()) { if (fileInfo.LastWriteTime < beforeDate) { fileInfo.Delete(); ++remove; } else { ++remain; } } table.Rows.Add(channelDirectory, remove, remain, remove + remain); totalRemoved += remove; } Spludlow.Log.Report("MetSat ClearImportDirectory; importSourceId:" + _ImportSourceId + ", daysHistory:" + daysHistory + ", totalRemoved: " + totalRemoved, table); } /// /// Delete old files based of LastWriteTime from the "Ready" directory /// Only un-implimented product files will build up in "Ready", implimented product files will be removed when they are processed /// public static void ClearReadyDirectory(int days) { DateTime beforeDate = DateTime.Now.Date.AddDays(-days); DataTable table = Spludlow.Data.TextTable.ReadText(new string[] { "Product TotalCount TotalLength DeleteCount DeleteLength", "String Int64 Int64 Int64 Int64", }); foreach (string productDirectory in Directory.GetDirectories(_ReadyDirectory)) { DirectoryInfo directoryInfo = new DirectoryInfo(productDirectory); string productName = Path.GetFileName(productDirectory); long totalCount = 0; long totalLength = 0; long deleteLength = 0; List deleteFilenames = new List(); foreach (FileInfo fileInfo in directoryInfo.GetFiles()) { ++totalCount; totalLength += fileInfo.Length; if (fileInfo.LastWriteTime < beforeDate) { deleteFilenames.Add(fileInfo.FullName); deleteLength += fileInfo.Length; } } table.Rows.Add(productName, totalCount, totalLength, deleteFilenames.Count, deleteLength); foreach (string filename in deleteFilenames) File.Delete(filename); } Spludlow.Log.Report("ClearReadyDirectory", table); } } }