// Spludlow Software
// Copyright © Samuel P. Ludlow 2020 All Rights Reserved
// Distributed under the terms of the GNU General Public License version 3
// Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE
// https://www.spludlow.co.uk/LICENCE.TXT
// The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission
// v1.14
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Data;
namespace Spludlow.MetSat
{
public class Receive
{
public static string _ReadyDirectory;
public static string _ImportSourceId;
public static Spludlow.Data.IDAL _Database;
///
/// Identify and Receive files from EUMETCast received directory into the "Ready"
///
static Receive()
{
_ReadyDirectory = Spludlow.Config.Get("MetSat.Ready");
_ImportSourceId = "EC";
_Database = Spludlow.Data.DAL.Create("@MetSat");
}
///
/// Identify and move files from EUMETCast received directory into the "Ready" product directories
/// Directory listing can be performed seperatly to increase speed in development when a lot of files are in EUMETCast received
///
public static void ReceiveDirectory()
{
string dropHost = null; // If source host is running Spludlow this can be set to speed up directory listing
Schema.ImportSourcesRow importSourceRow = ImportSourcesRow(_ImportSourceId);
using (Spludlow.TempDirectory tempDir = new TempDirectory())
{
string cacheInfoDirectory = tempDir + @"\DataSet";
SaveDirectory(dropHost, importSourceRow.Path, cacheInfoDirectory);
ReceiveDirectory(cacheInfoDirectory, _ReadyDirectory, true);
}
}
public static Schema.ImportSourcesRow ImportSourcesRow(string importSourceId)
{
Schema.ImportSourcesDataTable importSourcesDataTable = new Schema.ImportSourcesDataTable();
_Database.Fill(importSourcesDataTable, "SELECT * FROM ImportSources WHERE (ImportSourceId = @ImportSourceId)", importSourceId);
if (importSourcesDataTable.Rows.Count != 1)
throw new ApplicationException("MetSat Receive, Can not find importSource Row: " + importSourceId);
Schema.ImportSourcesRow importSourceRow = (Schema.ImportSourcesRow)importSourcesDataTable.Rows[0];
return importSourceRow;
}
public static int SaveDirectory(string host, string directory, string cacheInfoDirectory)
{
if (Directory.Exists(cacheInfoDirectory) == true)
Directory.Delete(cacheInfoDirectory, true);
DataSet dirDataSet;
if (host == null)
dirDataSet = Spludlow.Io.DirectoryList.List(directory, true, true);
else
dirDataSet = Spludlow.Io.DirectoryList.List(host, directory, true, true);
Spludlow.Data.TextTable.WriteDirectory(cacheInfoDirectory, dirDataSet);
return dirDataSet.Tables[0].Rows.Count;
}
public static void ReadDirectory(string cacheInfoDirectory)
{
ReceiveDirectory(cacheInfoDirectory, null, false);
}
public static void ReceiveDirectory(string cacheInfoDirectory, string readyDirectory, bool deleteSource)
{
DataSet dataSet;
dataSet = Spludlow.Data.TextTable.ReadDirectory(cacheInfoDirectory);
Spludlow.SysTables.FileSystemInfosDataTable fileSystemTable = new SysTables.FileSystemInfosDataTable();
foreach (DataRow row in dataSet.Tables[0].Rows)
fileSystemTable.ImportRow(row);
fileSystemTable.Columns.Add("ProductId", typeof(string));
DataTable channelTable = Spludlow.Data.TextTable.ReadText("Channels", new string[]
{
"ChannelDirectory ChannelName DirectoryId FileCount FileTotalSize FileTotalSizeText FileMinDate FileMaxDate SubDirCount TakeCount LeaveCount ErrorCount Products",
"String String Int32 Int32 Int64 String DateTime DateTime Int32 Int32 Int32 Int32 String",
});
foreach (Spludlow.SysTables.FileSystemInfosRow channelDirectoryRow in fileSystemTable.Select("Type = 'D' AND Depth = 1"))
{
string channelDirectory = channelDirectoryRow.Path;
string channelName = Path.GetFileName(channelDirectory);
DataRow channelRow = channelTable.Rows.Add(channelDirectory, channelName, channelDirectoryRow.DirectoryId);
DataRow[] fileRows = fileSystemTable.Select("Type = 'F' AND ParentDirectoryId = " + channelDirectoryRow.DirectoryId);
DateTime minDate = DateTime.MaxValue;
DateTime maxDate = DateTime.MinValue;
long totalFileSize = 0;
Dictionary products = new Dictionary();
foreach (Spludlow.SysTables.FileSystemInfosRow fileRow in fileRows)
{
if (fileRow.LastWriteTime < minDate)
minDate = fileRow.LastWriteTime;
if (fileRow.LastWriteTime > maxDate)
maxDate = fileRow.LastWriteTime;
totalFileSize += (long)fileRow.Length;
string name = Path.GetFileName(fileRow.Path);
string product = Spludlow.MetSat.ProductIdentify.DetermineProduct(name);
fileRow["ProductId"] = product;
if (products.ContainsKey(product) == false)
products.Add(product, 0);
++products[product];
}
StringBuilder productText = new StringBuilder();
foreach (string product in products.Keys)
{
if (productText.Length > 0)
productText.Append(", ");
productText.Append(product + ":" + products[product]);
}
channelRow["Products"] = productText.ToString();
if (fileRows.Length > 0)
{
channelRow["FileCount"] = fileRows.Length;
channelRow["FileTotalSize"] = totalFileSize;
channelRow["FileTotalSizeText"] = Spludlow.Text.DataSize(totalFileSize);
channelRow["FileMinDate"] = minDate;
channelRow["FileMaxDate"] = maxDate;
}
DataRow[] directoryRows = fileSystemTable.Select("Type = 'D' AND ParentDirectoryId = " + channelDirectoryRow.DirectoryId);
channelRow["SubDirCount"] = directoryRows.Length;
}
if (readyDirectory != null)
{
foreach (DataRow channelRow in channelTable.Rows)
{
string channelDirectory = (string)channelRow["ChannelDirectory"];
string channelName = (string)channelRow["ChannelName"];
int directoryId = (int)channelRow["DirectoryId"];
DataRow[] fileRows = fileSystemTable.Select("Type = 'F' AND ParentDirectoryId = " + directoryId);
int takeCount = 0;
int leaveCount = 0;
int errorCount = 0;
HashSet checkedProductDirs = new HashSet();
using (Spludlow.TempDirectory tempDir = new TempDirectory())
{
string arcDirectory = tempDir.Path + @"\ARC";
Directory.CreateDirectory(arcDirectory);
foreach (Spludlow.SysTables.FileSystemInfosRow fileRow in fileRows)
{
string productId = (string)fileRow["ProductId"];
if (productId == "")
{
++leaveCount;
continue;
}
if (checkedProductDirs.Contains(productId) == false)
{
string targetDirectory = readyDirectory + @"\" + productId;
if (Directory.Exists(targetDirectory) == false)
Directory.CreateDirectory(targetDirectory);
checkedProductDirs.Add(productId);
}
string extractFilename = "";
try
{
extractFilename = ExtractFile(fileRow.Path, tempDir.Path, arcDirectory);
string sourceFilename = (extractFilename != null ? extractFilename : fileRow.Path);
string name = Path.GetFileName(sourceFilename);
StringBuilder filename = new StringBuilder();
filename.Append(readyDirectory);
filename.Append(@"\");
filename.Append(productId);
filename.Append(@"\");
filename.Append(name);
string targetFilename = filename.ToString();
if (File.Exists(targetFilename) == true)
{
File.Delete(targetFilename);
Spludlow.Log.Warning("MetSat Receive, ReceiveDirectory; Target file already exists, will replace, source:" + sourceFilename + " target:" + targetFilename);
}
File.Copy(sourceFilename, targetFilename);
File.SetLastWriteTime(targetFilename, fileRow.LastWriteTime);
if (deleteSource == true)
File.Delete(fileRow.Path);
++takeCount;
}
catch (Exception ee)
{
Spludlow.Log.Error("MetSat Receive, ReceiveDirectory; Source File: " + fileRow.Path, ee);
++errorCount;
}
if (extractFilename != null)
{
Directory.Delete(arcDirectory, true);
Directory.CreateDirectory(arcDirectory);
}
}
}
channelRow["TakeCount"] = takeCount;
channelRow["LeaveCount"] = leaveCount;
channelRow["ErrorCount"] = errorCount;
}
}
dataSet = new DataSet();
dataSet.Tables.Add(channelTable);
Spludlow.Log.Report("MetSat Receive, ReceiveDirectory: " + cacheInfoDirectory, dataSet);
}
private static string ExtractFile(string filename, string tempRootDirectory, string arcDirectory)
{
string name = Path.GetFileName(filename);
if ((name.StartsWith("L-") == true || name.StartsWith("H-")) && name[name.Length - 2] == 'C')
{
if (filename.Contains(@":\") == true)
return Spludlow.MetSat.Xrit2Pic.XRITDecompress(filename, arcDirectory, true);
string tempFilename = tempRootDirectory + @"\" + name;
File.Copy(filename, tempFilename);
try
{
return Spludlow.MetSat.Xrit2Pic.XRITDecompress(tempFilename, arcDirectory, true);
}
finally
{
File.Delete(tempFilename);
}
}
string extention = Path.GetExtension(filename).ToLower();
if (extention == ".bz2" || extention == ".gz")
{
Spludlow.Archive.Extract(filename, arcDirectory);
string[] results = Directory.GetFiles(arcDirectory);
if (results.Length == 1)
return results[0];
if (results.Length > 1)
throw new ApplicationException("ReceiveFile Extract, More than one file: " + filename);
results = Directory.GetFiles(arcDirectory, "*", SearchOption.AllDirectories);
if (results.Length != 1)
throw new ApplicationException("ReceiveFile Extract, Not one deep file: " + filename + ", count: " + results.Length);
return results[0];
}
return null;
}
///
/// Delete old files based of LastWriteTime from the EUMETCast received directory
/// Only un-identified product files will build up in the received directory, identified product files will have been moved to "Ready"
///
public static void ClearReceivedDirectory(int daysHistory)
{
DateTime beforeDate = DateTime.Now.Date.AddDays(-daysHistory);
DataTable table = Spludlow.Data.TextTable.ReadText(new string[]
{
"ChannelDirectory Removed Remaining Total",
"String Int32 Int32 Int32",
});
Schema.ImportSourcesRow importSourceRow = ImportSourcesRow(_ImportSourceId);
if (importSourceRow.ImportSourceTypeId != "WD")
throw new ApplicationException("MetSat ClearImportDirectory; Not a WD Import Source: " + importSourceRow.ImportSourceId);
int totalRemoved = 0;
foreach (string channelDirectory in Directory.GetDirectories(importSourceRow.Path))
{
int remove = 0;
int remain = 0;
DirectoryInfo dirInfo = new DirectoryInfo(channelDirectory);
foreach (FileInfo fileInfo in dirInfo.GetFiles())
{
if (fileInfo.LastWriteTime < beforeDate)
{
fileInfo.Delete();
++remove;
}
else
{
++remain;
}
}
table.Rows.Add(channelDirectory, remove, remain, remove + remain);
totalRemoved += remove;
}
Spludlow.Log.Report("MetSat ClearImportDirectory; importSourceId:" + _ImportSourceId + ", daysHistory:" + daysHistory + ", totalRemoved: " + totalRemoved, table);
}
///
/// Delete old files based of LastWriteTime from the "Ready" directory
/// Only un-implimented product files will build up in "Ready", implimented product files will be removed when they are processed
///
public static void ClearReadyDirectory(int days)
{
DateTime beforeDate = DateTime.Now.Date.AddDays(-days);
DataTable table = Spludlow.Data.TextTable.ReadText(new string[]
{
"Product TotalCount TotalLength DeleteCount DeleteLength",
"String Int64 Int64 Int64 Int64",
});
foreach (string productDirectory in Directory.GetDirectories(_ReadyDirectory))
{
DirectoryInfo directoryInfo = new DirectoryInfo(productDirectory);
string productName = Path.GetFileName(productDirectory);
long totalCount = 0;
long totalLength = 0;
long deleteLength = 0;
List deleteFilenames = new List();
foreach (FileInfo fileInfo in directoryInfo.GetFiles())
{
++totalCount;
totalLength += fileInfo.Length;
if (fileInfo.LastWriteTime < beforeDate)
{
deleteFilenames.Add(fileInfo.FullName);
deleteLength += fileInfo.Length;
}
}
table.Rows.Add(productName, totalCount, totalLength, deleteFilenames.Count, deleteLength);
foreach (string filename in deleteFilenames)
File.Delete(filename);
}
Spludlow.Log.Report("ClearReadyDirectory", table);
}
}
}