// Spludlow Software
// Copyright © Samuel P. Ludlow 2020 All Rights Reserved
// Distributed under the terms of the GNU General Public License version 3
// Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE
// https://www.spludlow.co.uk/LICENCE.TXT
// The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission
// v1.14
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Data;
using System.Drawing;
using System.Drawing.Imaging;
namespace Spludlow.MetSat
{
///
/// XRIT Handling
/// Get the bitmaps out of the data files
///
public class XRIT
{
public class XritFile
{
public PrimaryHeaderRecord Header;
public ImageStructureRecord Structure;
public ImageNavigationRecord Navigation;
public AnnotationRecord Annotation;
public TimeStampRecord TimeStamp;
public SegmentIdentificationRecord SegmentIdentification;
public byte[] Data;
public byte[] HeaderTypes;
}
public class PrimaryHeaderRecord
{
public Byte HeaderType; // 1 integer, unsigned header type, set to 0
public UInt16 HeaderRecordLength; // 2 integer, unsigned header record length, set to 16 (bytes)
public Byte FileTypeCode; // 1 integer, unsigned file type code, determining the top level structure of the file data field
public UInt32 TotalHeaderLength; // 4 integer, unsigned total header length, specifying the total size of all header records (including this one) in octets
public UInt64 DataFieldLength; // 8 integer, unsigned data field length, specifying the total size of the file data field in bits.
}
public class ImageStructureRecord
{
public Byte BitsPerPixel; // 1 integer, unsigned number of bits per pixel (1 ... 255) NB
public UInt16 Columns; // 2 integer, unsigned number of columns (1 ... 65535) NC
public UInt16 Lines; // 2 integer, unsigned number of lines (1 ... 65535) NL
public Byte Compression; // 1 integer, unsigned compression flag (0,1,2) CFLG
}
public class ImageNavigationRecord
{
public String ProjectionName; // 32 character projection name
public Int32 ColumnScalingFactor; // 4 integer, signed column scaling factor CFAC
public Int32 LineScalingFactor; // 4 integer, signed line scaling factor LFAC
public Int32 ColumnOffset; // 4 integer, signed column offset COFF
public Int32 LineOffset; // 4 integer, signed line offset LOFF
}
public static int[] ImageNavigationRecord_StringLengths = new int[] { 32 };
public class AnnotationRecord
{
public String XRITchannelID;
public String FieldSeparator0;
public String DisseminationID;
public String FieldSeparator1;
public String DisseminatingSC;
public String FieldSeparator2;
public String ProductID1;
public String FieldSeparator3;
public String ProductID2;
public String FieldSeparator4;
public String ProductID3;
public String FieldSeparator5;
public String ProductID4;
public String FieldSeparator6;
public String Flags;
}
public static int[] AnnotationRecord_StringLengths = new int[] { 1, 1, 3, 1, 6, 1, 12, 1, 9, 1, 9, 1, 12, 1, 2 };
public class TimeStampRecord
{
public Byte CDS_P_Field;
public UInt16 CDS_T_Field_Days; // 2 bytes counter of days starting from 1 January 1958
public UInt32 CDS_T_Field_Milliseconds; // 4 bytes milliseconds of day
}
public class SegmentIdentificationRecord
{
public UInt16 GP_SC_ID;
public Byte Spectral_Channel_ID;
public UInt16 Segm_Seq_No;
public UInt16 Planned_Start_Segm_Seq_No;
public UInt16 Planned_End_Segm_Seq_No;
public Byte Representation;
}
// FileTypeCode
// 0 image data file
// 1 GTS message
// 2 alphanumeric text file
// 3 encryption key message
// 4...127 reserved for future global usage
// 128...255 for mission specific use
// HeaderType
// 0 primary header all -
// 1 image structure 0 -
// 2 image navigation - 0
// 3 image data function - 0
// 4 annotation - all
// 5 time stamp - all
// 6 ancillary text - all
// 7 key header - all
// 8...127 reserved for future global usage
// 128...255 for mission specific use
private static bool Endian = true;
private static PixelFormat DefaultPixelFormat = PixelFormat.Format8bppIndexed;
private static Spludlow.MetSat.DataItems DataItems = new DataItems();
private static string[] fileTypesLookupData = new string[]
{
"0 Image Data",
"1 GTS Message",
"2 Alphanumeric Text",
"128 Repeat Cycle Prologue",
"129 Repeat Cycle Epilogue",
};
private static string[] headerTypesLookupData = new string[] // EPI & PRO only got: 4, 5
{
"1 Image Structure",
"2 Image Navigation",
"4 Annotation",
"5 Time Stamp",
"128 Image Segment Identification", // Segment Identification header will define the first and the last segment number of the image segments forming part of a repeat cycle
"129 Image Segment Line Quality",
};
private static DataTable ChannelVaribles = Spludlow.Data.TextTable.ReadText(new string []
{
"DataProductId Channel Palette Brightness",
"String* String* String Double",
"EO_EUM_DAT_MSG_LRSEVIRI IR_016 Blue-Green-2White 1.9",
"EO_EUM_DAT_MSG_LRSEVIRI IR_039 BlackRedYellowWhite 1.9",
"EO_EUM_DAT_MSG_LRSEVIRI IR_108 BlackRedYellowWhite 1.6",
"EO_EUM_DAT_MSG_LRSEVIRI VIS006 Blue-Green-2White 1.4",
"EO_EUM_DAT_MSG_LRSEVIRI WV_062 BlueCyan 1.3",
"EO_EUM_DAT_MSG_HRSEVIRI HRV Blue-Green-4White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI IR_016 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI IR_039 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI IR_087 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI IR_097 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI IR_108 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI IR_120 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI IR_134 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI VIS006 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI VIS008 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI WV_062 BlueCyan 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI WV_073 GreenYellow 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC HRV Blue-Green-4White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_016 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_039 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_087 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_097 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_108 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_120 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC IR_134 BlackRedYellowWhite 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC VIS006 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC VIS008 Blue-Green-2White 0.4",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC WV_062 BlueCyan 0.3",
"EO_EUM_DAT_MSG_HRSEVIRI-IODC WV_073 GreenYellow 0.4",
});
public static void Report(string directory)
{
string[] columnLayouts = new string[]
{
"CId 0 1",
"DId 2 3",
"DSCId 6 6",
"ProductID1 13 12",
"ProductID2 26 9", // Channel
"ProductID3 36 9", // File Sequance
"ProductID4 46 12", // SensingTime
"ext 58 3",
};
string[] filenames = Directory.GetFiles(directory);
DataTable table = Spludlow.MetSat.Process.LoadFilenameInfo(columnLayouts, filenames);
table.Columns.Add("SatIntDesId", typeof(string));
table.Columns.Add("SensingTime", typeof(DateTime));
table.Columns.Add("Channel", typeof(string));
table.Columns.Add("Sequence", typeof(string));
table.Columns.Add("Group", typeof(string));
table.Columns.Add("FileTypeCode", typeof(byte));
table.Columns.Add("DataFieldLength", typeof(ulong));
table.Columns.Add("GP_SC_ID", typeof(UInt16));
table.Columns.Add("Spectral_Channel_ID", typeof(Byte));
table.Columns.Add("Segm_Seq_No", typeof(UInt16));
table.Columns.Add("Planned_Start_Segm_Seq_No", typeof(UInt16));
table.Columns.Add("Planned_End_Segm_Seq_No", typeof(UInt16));
table.Columns.Add("NavigationColumnOffset", typeof(int));
table.Columns.Add("NavigationLineOffset", typeof(int));
table.Columns.Add("StructureBitsPerPixel", typeof(byte));
table.Columns.Add("StructureColumns", typeof(ushort));
table.Columns.Add("StructureLines", typeof(ushort));
table.Columns.Add("HeaderTypes", typeof(string));
List groups = new List();
foreach (DataRow row in table.Rows)
{
string satId = (string)row["ProductID1"];
if (satId.StartsWith("MPEF_") == true)
satId = (string)row["DSCId"];
string satIntDesId = satId;
string senseTime = (string)row["ProductID4"];
string group = satIntDesId + "-" + senseTime;
row["SatIntDesId"] = satIntDesId;
row["SensingTime"] = DateTime.ParseExact(senseTime, "yyyyMMddHHmm", null);
row["Channel"] = ((string)row["ProductID2"]).Trim(new char[] { '_' });
row["Sequence"] = ((string)row["ProductID3"]).Trim(new char[] { '_' });
row["Group"] = group;
if (groups.Contains(group) == false)
groups.Add(group);
}
groups.Sort();
List groupTables = new List();
for (int groupIndex = 0; groupIndex < groups.Count; ++groupIndex)
{
string group = groups[groupIndex];
DataTable groupTable = table.Clone();
foreach (DataRow row in table.Select("Group = '" + group + "'", "Channel, Sequence"))
groupTable.ImportRow(row);
groupTables.Add(groupTable);
foreach (DataRow row in groupTable.Rows)
{
string filename = (string)row["Filename"];
XritFile xritFile = LoadFile(filename);
row["FileTypeCode"] = xritFile.Header.FileTypeCode;
row["DataFieldLength"] = xritFile.Header.DataFieldLength;
if (xritFile.SegmentIdentification != null)
{
row["GP_SC_ID"] = xritFile.SegmentIdentification.GP_SC_ID;
row["Spectral_Channel_ID"] = xritFile.SegmentIdentification.Spectral_Channel_ID;
row["Segm_Seq_No"] = xritFile.SegmentIdentification.Segm_Seq_No;
row["Planned_Start_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_Start_Segm_Seq_No;
row["Planned_End_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_End_Segm_Seq_No;
}
if (xritFile.Navigation != null)
{
row["NavigationColumnOffset"] = xritFile.Navigation.ColumnOffset;
row["NavigationLineOffset"] = xritFile.Navigation.LineOffset;
}
if (xritFile.Structure != null)
{
row["StructureBitsPerPixel"] = xritFile.Structure.BitsPerPixel;
row["StructureColumns"] = xritFile.Structure.Columns;
row["StructureLines"] = xritFile.Structure.Lines;
}
StringBuilder headerTypes = new StringBuilder();
foreach (byte headerType in xritFile.HeaderTypes)
{
if (headerTypes.Length > 0)
headerTypes.Append(", ");
headerTypes.Append(headerType);
}
row["HeaderTypes"] = headerTypes.ToString();
}
bool groupComlete = IsComplete(groupTable);
// If not complete and on last group then skip
if (groupComlete == false && (groupIndex == (groups.Count - 1)))
{
Spludlow.Log.Warning("XRIT Last Group Incomplete skipping: " + group);
continue;
}
Spludlow.Log.Info("Complete " + group + " " + groupComlete.ToString());
}
Spludlow.Log.Report("Report", groups.ToArray(), groupTables.ToArray()); //, table);
}
public static bool IsComplete(DataTable groupTable)
{
DataRow[] rows;
rows = groupTable.Select("FileTypeCode = 129");
if (rows.Length == 1)
return true;
if (rows.Length > 1)
throw new ApplicationException("XRIT IsComplete Group table has more than one EPI(129)");
List channels = new List();
foreach (DataRow row in groupTable.Rows)
{
if (row.IsNull("Spectral_Channel_ID") == true)
continue;
int channel = (byte)row["Spectral_Channel_ID"];
if (channels.Contains(channel) == false)
channels.Add(channel);
}
if (channels.Count == 0)
return false;
foreach (int channel in channels)
{
rows = groupTable.Select("Spectral_Channel_ID = " + channel);
int last = (UInt16)rows[0]["Planned_End_Segm_Seq_No"];
if (rows.Length != last)
return false;
}
return true;
}
public static void Run(string directory)
{
string dataProductId = Path.GetFileName(directory).ToUpper();
bool grib = false;
if (dataProductId == "EO_EUM_DAT_MSG_CLM")
grib = true;
if (dataProductId == "EO_EUM_DAT_MSG_CLM-IODC")
grib = true;
string[] columnLayouts = new string[]
{
"CId 0 1",
"DId 2 3",
"DSCId 6 6",
"ProductID1 13 12",
"ProductID2 26 9", // Channel
"ProductID3 36 9", // File Sequance
"ProductID4 46 12", // SensingTime
"ext 58 3",
};
string[] filenames = Directory.GetFiles(directory);
DataTable table = Spludlow.MetSat.Process.LoadFilenameInfo(columnLayouts, filenames);
table.Columns.Add("SatIntDesId", typeof(string));
table.Columns.Add("SensingTime", typeof(DateTime));
table.Columns.Add("Channel", typeof(string));
table.Columns.Add("Sequence", typeof(string));
table.Columns.Add("Group", typeof(string));
table.Columns.Add("FileTypeCode", typeof(byte));
table.Columns.Add("DataFieldLength", typeof(ulong));
table.Columns.Add("GP_SC_ID", typeof(UInt16));
table.Columns.Add("Spectral_Channel_ID", typeof(Byte));
table.Columns.Add("Segm_Seq_No", typeof(UInt16));
table.Columns.Add("Planned_Start_Segm_Seq_No", typeof(UInt16));
table.Columns.Add("Planned_End_Segm_Seq_No", typeof(UInt16));
table.Columns.Add("NavigationColumnOffset", typeof(int));
table.Columns.Add("NavigationLineOffset", typeof(int));
table.Columns.Add("StructureBitsPerPixel", typeof(byte));
table.Columns.Add("StructureColumns", typeof(ushort));
table.Columns.Add("StructureLines", typeof(ushort));
table.Columns.Add("HeaderTypes", typeof(string));
List groups = new List();
foreach (DataRow row in table.Rows)
{
string satId = (string)row["ProductID1"];
if (satId.StartsWith("MPEF_") == true)
satId = (string)row["DSCId"];
string satIntDesId = satId;
string senseTime = (string)row["ProductID4"];
string group = satIntDesId + "-" + senseTime;
row["SatIntDesId"] = satIntDesId;
row["SensingTime"] = DateTime.ParseExact(senseTime, "yyyyMMddHHmm", null);
row["Channel"] = ((string)row["ProductID2"]).Trim(new char[] { '_' });
row["Sequence"] = ((string)row["ProductID3"]).Trim(new char[] { '_' });
row["Group"] = group;
if (groups.Contains(group) == false)
groups.Add(group);
}
groups.Sort();
Spludlow.MetSat.DataItems dataItems = new Spludlow.MetSat.DataItems();
for (int groupIndex = 0; groupIndex < groups.Count; ++groupIndex)
{
string group = groups[groupIndex];
DataTable groupTable = table.Clone();
foreach (DataRow row in table.Select("Group = '" + group + "'", "Channel, Sequence"))
groupTable.ImportRow(row);
Dictionary xritFiles = new Dictionary();
foreach (DataRow row in groupTable.Rows)
{
string filename = (string)row["Filename"];
XritFile xritFile = LoadFile(filename);
xritFiles.Add(filename, xritFile);
row["FileTypeCode"] = xritFile.Header.FileTypeCode;
row["DataFieldLength"] = xritFile.Header.DataFieldLength;
if (xritFile.SegmentIdentification != null)
{
row["GP_SC_ID"] = xritFile.SegmentIdentification.GP_SC_ID;
row["Spectral_Channel_ID"] = xritFile.SegmentIdentification.Spectral_Channel_ID;
row["Segm_Seq_No"] = xritFile.SegmentIdentification.Segm_Seq_No;
row["Planned_Start_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_Start_Segm_Seq_No;
row["Planned_End_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_End_Segm_Seq_No;
}
if (xritFile.Navigation != null)
{
row["NavigationColumnOffset"] = xritFile.Navigation.ColumnOffset;
row["NavigationLineOffset"] = xritFile.Navigation.LineOffset;
}
if (xritFile.Structure != null)
{
row["StructureBitsPerPixel"] = xritFile.Structure.BitsPerPixel;
row["StructureColumns"] = xritFile.Structure.Columns;
row["StructureLines"] = xritFile.Structure.Lines;
}
StringBuilder headerTypes = new StringBuilder();
foreach (byte headerType in xritFile.HeaderTypes)
{
if (headerTypes.Length > 0)
headerTypes.Append(", ");
headerTypes.Append(headerType);
}
row["HeaderTypes"] = headerTypes.ToString();
}
bool groupComlete = IsComplete(groupTable);
// If not complete and on last group then skip
if (groupComlete == false && (groupIndex == (groups.Count - 1)))
{
Spludlow.Log.Warning("XRIT Group Incomplete skipping: " + dataProductId + " " + directory + ", " + group, groupTable);
continue;
}
if (grib == false)
LoadGroup(groupTable, dataProductId, dataItems, xritFiles);
else
LoadGribGroup(groupTable, dataProductId, dataItems, xritFiles);
Spludlow.Log.Report(dataProductId + " " + directory + ", " + group, groupTable);
}
Spludlow.Log.Finish(dataProductId + " Finished; Groups:" + groups.Count + ", Files:" + table.Rows.Count);
}
private static void LoadGribGroup(DataTable groupTable, string dataProductId, Spludlow.MetSat.DataItems dataItems, Dictionary xritFiles)
{
List xrits = new List();
DataRow firstRow = null;
int dataSize = 0;
List sourceFilenames = new List();
foreach (DataRow row in groupTable.Rows)
{
string filename = (string)row["Filename"];
string sequenceText = (string)row["Sequence"];
try
{
int sequence = 0;
if (sequenceText[0] == '0')
sequence = Int32.Parse(sequenceText);
if (sequence > 0)
{
if (firstRow == null)
firstRow = row;
XritFile info = xritFiles[filename];
xrits.Add(info);
dataSize += info.Data.Length;
}
}
catch (Exception ee)
{
throw new ApplicationException("Loading File :\t" + filename + "\t" + ee.Message, ee);
}
sourceFilenames.Add(filename);
}
if (firstRow != null)
{
Schema.DataItemFilesDataTable filesTable = new Schema.DataItemFilesDataTable();
filesTable.DataItemFileIdColumn.ReadOnly = false;
filesTable.DataItemIdColumn.AllowDBNull = true;
string satIntDesId = (string)firstRow["SatIntDesId"];
DateTime sensingTime = (DateTime)firstRow["SensingTime"];
string dataGroupName = "SatId" + " - " + "SEVIRI" + " - " + Spludlow.Text.TimeStamp(sensingTime);
using (Spludlow.TempDirectory tempDir = new TempDirectory())
{
Schema.DataItemFilesRow fileRow;
// Source Archive
string name = dataGroupName + "-SourceArchive";
string tempCopyDirectory = tempDir + @"\" + name;
Directory.CreateDirectory(tempCopyDirectory);
foreach (string sourceFilename in sourceFilenames)
File.Copy(sourceFilename, tempCopyDirectory + @"\" + Path.GetFileName(sourceFilename));
fileRow = dataItems.MakeFileRow(filesTable, "SA", name + ".7z", "");
string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });
Archive.Create(tempStoreName[0], tempCopyDirectory, true);
// Bitmap
string channel = "CLM"; // !!!!!!!!!!!!!!!!!!!!!! may be others not supported yet
name = dataGroupName + "-" + channel;
byte[] gribData = new byte[dataSize];
int offset = 0;
foreach (XritFile xrit in xrits)
{
Buffer.BlockCopy(xrit.Data, 0, gribData, offset, xrit.Data.Length);
offset += xrit.Data.Length;
}
try
{
Spludlow.MetSat.GRIB grib = new GRIB(gribData);
fileRow = dataItems.MakeFileRow(filesTable, "CH", name + "-" + channel + ".png", channel);
tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });
Spludlow.MetSat.CLM.ProcessGRIB(grib, tempStoreName[0]);
Size size = Spludlow.Drawing.Bitmaps.Dimensions(tempStoreName[0]);
fileRow.Size = (int)Spludlow.Io.Files.FileLength(tempStoreName[0]);
fileRow.Width = size.Width;
fileRow.Height = size.Height;
fileRow.Pixels = size.Width * size.Height;
}
catch (Exception ee)
{
Spludlow.Log.Warning("MetSat XRIT Processing GRIB data: " + name, ee);
}
// Save
dataItems.SaveDataItem(filesTable, satIntDesId, dataProductId, sensingTime, dataGroupName);
}
}
foreach (string sourceFilename in sourceFilenames)
File.Delete(sourceFilename);
}
private static void LoadGroup(DataTable groupTable, string dataProductId, Spludlow.MetSat.DataItems dataItems, Dictionary xritFiles)
{
Dictionary> channelGroupData = new Dictionary>();
DataRow firstRow = null;
List sourceFilenames = new List();
foreach (DataRow row in groupTable.Rows)
{
string filename = (string)row["Filename"];
string channel = (string)row["Channel"];
string sequenceText = (string)row["Sequence"];
try
{
int sequence = 0;
if (sequenceText[0] == '0')
sequence = Int32.Parse(sequenceText);
if (sequence > 0)
{
if (firstRow == null)
firstRow = row;
XritFile info = xritFiles[filename];
if (channelGroupData.ContainsKey(channel) == false)
channelGroupData[channel] = new List();
channelGroupData[channel].Add(info);
}
}
catch (Exception ee)
{
throw new ApplicationException("Loading File :\t" + filename + "\t" + ee.Message, ee);
}
sourceFilenames.Add(filename);
}
if (firstRow != null)
{
Schema.DataItemFilesDataTable filesTable = new Schema.DataItemFilesDataTable();
filesTable.DataItemFileIdColumn.ReadOnly = false;
filesTable.DataItemIdColumn.AllowDBNull = true;
string satIntDesId = (string)firstRow["SatIntDesId"];
//string filename = Path.GetFileName((string)firstRow["Filename"]);
DateTime sensingTime = (DateTime)firstRow["SensingTime"];
// Spludlow.MetSat.Satellites.SatelliteName(satIntDesId)
string dataGroupName = "SatId" + " - " + "SEVIRI" + " - " + Spludlow.Text.TimeStamp(sensingTime);
using (Spludlow.TempDirectory tempDir = new TempDirectory())
{
Schema.DataItemFilesRow fileRow;
// Source Archive
string name = dataGroupName + "-SourceArchive";
string tempCopyDirectory = tempDir + @"\" + name;
Directory.CreateDirectory(tempCopyDirectory);
foreach (string sourceFilename in sourceFilenames)
File.Copy(sourceFilename, tempCopyDirectory + @"\" + Path.GetFileName(sourceFilename));
fileRow = dataItems.MakeFileRow(filesTable, "SA", name + ".7z", "");
string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });
Archive.Create(tempStoreName[0], tempCopyDirectory, true);
//Spludlow.Log.Report(dataGroupName, groupTable);
// Channel Bitmpas
OutputChannelBitmaps(channelGroupData, dataGroupName + "-Channel", dataItems, filesTable, dataProductId);
// Save
dataItems.SaveDataItem(filesTable, satIntDesId, dataProductId, sensingTime, dataGroupName);
}
}
foreach (string sourceFilename in sourceFilenames)
File.Delete(sourceFilename);
}
public static void Draw(XritFile xritFile, Bitmap bitmap, int yOffset, double scale)
{
byte[] bitmapData = Spludlow.Drawing.Bitmaps.ReadBitmapData(bitmap, DefaultPixelFormat);
int[] values = Values(xritFile);
for (int y = 0; y < xritFile.Structure.Lines; ++y)
{
for (int x = 0; x < xritFile.Structure.Columns; ++x)
{
int index = (y * xritFile.Structure.Columns) + x;
int targetSegmentY = (xritFile.Structure.Lines - 1) - y;
int targetX = ((xritFile.Structure.Columns - 1) - x);
int destinationIndex = ((targetSegmentY + yOffset) * xritFile.Structure.Columns) + targetX;
int value = values[index];
if (scale != 1)
{
value = (int)Math.Round((double)value * scale, 0);
if (value > 0xFF)
value = 0xFF;
}
bitmapData[destinationIndex] = (byte)value;
}
}
Spludlow.Drawing.Bitmaps.WriteBitmapData(bitmap, bitmapData, DefaultPixelFormat);
}
private static void OutputChannelBitmaps(Dictionary> channelGroupData, string name, Spludlow.MetSat.DataItems dataItems, Schema.DataItemFilesDataTable filesTable, string dataProductId)
{
int highShadeMaxCount = 100;
Spludlow.Drawing.ACT act = new Drawing.ACT();
foreach (string channel in channelGroupData.Keys)
{
string palette = "grey";
double brightness = 1.0;
DataRow channelRow = ChannelVaribles.Rows.Find(new object[] { dataProductId, channel });
if (channelRow != null)
{
palette = (string)channelRow["Palette"];
brightness = (double)channelRow["Brightness"];
}
else
{
Spludlow.Log.Warning("XRIT; OutputChannelBitmaps, ChannelVaribles row not found:\t" + dataProductId + ", " + channel); // Alow not present for deafults
}
Dictionary histogram = new Dictionary();
foreach (XritFile info in channelGroupData[channel])
{
if (brightness == 0 && info.Header.FileTypeCode == 0)
{
int[] values = Values(info); // when using hostogram regetting values below !!!!!!!!!
foreach (int value in values)
{
if (histogram.ContainsKey(value) == false)
histogram.Add(value, 1);
else
histogram[value] = histogram[value] + 1;
}
}
}
if (brightness == 0)
{
DataTable histogramTable = Spludlow.Data.TextTable.ReadText(new string[]
{
"Value Count",
"Int32* Int32",
});
foreach (int value in histogram.Keys)
histogramTable.Rows.Add(new object[] { value, histogram[value] });
DataView view = new DataView(histogramTable);
view.Sort = "Value";
int high = (int)view[view.Count -1].Row["Value"];
for (int viewIndex = view.Count - 1; viewIndex > 0 && (int)view[viewIndex].Row["Count"] < highShadeMaxCount; --viewIndex)
{
high = (int)view[viewIndex].Row["Value"];
}
double scale = (double)0xFF / (double)high;
Spludlow.Log.Report("XRIT Histogram;\t" + name + "-" + channel + ", High:" + high + ", Scale:" + scale, new object[] { view });
brightness = scale;
}
XritFile xritFirst = channelGroupData[channel][0];
int imageWidth = xritFirst.Structure.Columns;
int imageHeight = xritFirst.Structure.Lines * xritFirst.SegmentIdentification.Planned_End_Segm_Seq_No;
int segmentHeight = xritFirst.Structure.Lines;
using (Bitmap bitmap = new Bitmap(imageWidth, imageHeight, DefaultPixelFormat))
{
if (palette == "grey")
{
Spludlow.Drawing.Bitmaps.SetPalette(bitmap);
}
else
{
byte[] paletteData = act.Get(palette);
paletteData[0] = 0;
paletteData[1] = 0;
paletteData[2] = 0;
Spludlow.Drawing.Bitmaps.SetPalette(bitmap, paletteData);
}
int moveDown = 0; // For some reason the HRV channel offsets are out by -2 ???
if (imageWidth != imageHeight)
moveDown = 2;
foreach (XritFile xritFile in channelGroupData[channel])
{
int yOffset = xritFile.Navigation.LineOffset + moveDown + (imageHeight / 2) - segmentHeight;
Draw(xritFile, bitmap, yOffset, brightness);
}
Schema.DataItemFilesRow fileRow = dataItems.MakeFileRow(filesTable, "CH", name + "-" + channel + ".png", channel);
string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });
bitmap.Save(tempStoreName[0], ImageFormat.Png);
fileRow.Size = (int)Spludlow.Io.Files.FileLength(tempStoreName[0]);
fileRow.Width = imageWidth;
fileRow.Height = imageHeight;
fileRow.Pixels = imageWidth * imageHeight;
}
}
}
public static XritFile LoadFile(string filename)
{
XritFile xritFile = new XritFile();
xritFile.Header = new PrimaryHeaderRecord();
using (FileStream stream = new FileStream(filename, FileMode.Open))
{
using (BinaryReader reader = new BinaryReader(stream))
{
long read;
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Header, Endian);
long headerLeft = xritFile.Header.TotalHeaderLength - read;
List headerTypeList = new List();
while (headerLeft > 0)
{
byte headerType = reader.ReadByte();
UInt16 headerRecordLength = Spludlow.Io.BinaryIO.ReadUInt16(reader, Endian);
headerTypeList.Add(headerType);
switch (headerType)
{
case 1:
xritFile.Structure = new ImageStructureRecord();
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Structure, Endian);
break;
case 2:
xritFile.Navigation = new ImageNavigationRecord();
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Navigation, Endian, ImageNavigationRecord_StringLengths);
break;
case 4:
xritFile.Annotation = new AnnotationRecord();
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Annotation, Endian, AnnotationRecord_StringLengths);
break;
case 5:
xritFile.TimeStamp = new TimeStampRecord();
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.TimeStamp, Endian);
break;
case 128:
xritFile.SegmentIdentification = new SegmentIdentificationRecord();
read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.SegmentIdentification, Endian);
break;
default:
read = headerRecordLength - 3;
reader.ReadBytes((int)read);
break;
}
if (read != (headerRecordLength - 3))
throw new ApplicationException("XRIT.LoadFile; Bad Header Size:\t" + filename);
headerLeft -= headerRecordLength;
}
if (headerLeft != 0)
throw new ApplicationException("XRIT.LoadFile; Bad Header Over Read:\t" + filename);
int dataSizeBytes = (int)(xritFile.Header.DataFieldLength / 8);
if ((xritFile.Header.DataFieldLength % 8) > 0)
++dataSizeBytes;
xritFile.Data = reader.ReadBytes(dataSizeBytes);
if (reader.BaseStream.Position != reader.BaseStream.Length)
throw new ApplicationException("XRIT.LoadFile; More data than expected past end of file:\t" + filename);
xritFile.HeaderTypes = headerTypeList.ToArray();
}
}
return xritFile;
}
public static DataTable Histogram(string[] filenames)
{
DataTable table = Spludlow.Data.TextTable.ReadText(new string[]
{
"Value Count",
"Int32* Int32",
});
foreach (string filename in filenames)
{
XritFile xritFile = LoadFile(filename);
if (xritFile.Header.FileTypeCode != 0)
continue;
foreach (int value in Values(xritFile))
{
DataRow row = table.Rows.Find(value);
if (row == null)
row = table.Rows.Add(new object[] { value, 0 });
row["Count"] = (int)row["Count"] + 1;
}
}
return table;
}
public static int[] Values(XritFile xritFile) //, double scale) // have scale only on other mathds . others will use values. dont re get for hostorgam
{
int total = xritFile.Structure.Columns * xritFile.Structure.Lines;
int[] values = new int[total];
for (int index = 0; index < total; ++index)
{
if (xritFile.Structure.BitsPerPixel == 8)
{
values[index] = xritFile.Data[index];
}
else
{
int bitIndex = index * xritFile.Structure.BitsPerPixel;
byte[] bytes = ExtractBytes(bitIndex, xritFile.Data);
int value = Spludlow.Io.BinaryIO.ReadUInt16(bytes, 0, Endian);
values[index] = value;
}
}
return values;
}
public static void Write(XritFile xritFile, string filename, int dpi)
{
using (Bitmap bitmap = new Bitmap(xritFile.Structure.Columns, xritFile.Structure.Lines, DefaultPixelFormat))
{
bitmap.SetResolution(dpi, dpi);
Spludlow.Drawing.Bitmaps.SetPalette(bitmap);
Draw(xritFile, bitmap, 0, 1);
bitmap.Save(filename, ImageFormat.Png);
}
}
public static byte[] ExtractBytes(int bitIndex, byte[] imageData)
{
// The bits of one pixel are oriented from MSB through LSB
// Only tested 10 bit
int byteIndex = bitIndex / 8;
int bitOffset = bitIndex % 8;
byte[] bytes = new byte[2];
bytes[0] = imageData[byteIndex];
bytes[1] = imageData[byteIndex + 1];
int shift = 6 - bitOffset;
if (bitOffset > 0)
{
// Blank out bits before offset in 1st byte
int mask = (int)Math.Pow(2, shift + 2) - 1;
bytes[0] = (byte)(bytes[0] & mask);
}
if (bytes[0] == 0 && bytes[1] == 0)
return bytes;
if (shift > 0)
{
// move along bits in 2nd byte
bytes[1] = (byte)(bytes[1] >> shift);
// copy bits from end of 1st byte into start of 2nd byte
int mask = (int)Math.Pow(2, shift) - 1;
byte temp = (byte)((bytes[0] & mask) << (bitOffset + 2));
bytes[1] = (byte)(bytes[1] | temp);
// move along start of 1st byte
bytes[0] = (byte)(bytes[0] >> shift);
}
return bytes;
}
}
}