//	Spludlow Software
//	Copyright © Samuel P. Ludlow 2020 All Rights Reserved
//	Distributed under the terms of the GNU General Public License version 3
//	Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE
//	https://www.spludlow.co.uk/LICENCE.TXT
//	The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission
//	v1.14

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using System.IO;
using System.Data;
using System.Drawing;
using System.Drawing.Imaging;

namespace Spludlow.MetSat
{
	/// <summary>
	/// XRIT Handling
	/// Get the bitmaps out of the data files
	/// </summary>
	public class XRIT
	{
		public class XritFile
		{
			public PrimaryHeaderRecord Header;

			public ImageStructureRecord Structure;
			public ImageNavigationRecord Navigation;

			public AnnotationRecord Annotation;
			public TimeStampRecord TimeStamp;
			public SegmentIdentificationRecord SegmentIdentification;

			public byte[] Data;

			public byte[] HeaderTypes;
		}

		public class PrimaryHeaderRecord
		{
			public Byte HeaderType;				//	1 integer, unsigned header type, set to 0
			public UInt16 HeaderRecordLength;	//	2 integer, unsigned header record length, set to 16 (bytes)
			public Byte FileTypeCode;			//	1 integer, unsigned file type code, determining the top level structure of the file data field
			public UInt32 TotalHeaderLength;	//	4 integer, unsigned total header length, specifying the total size of all header records (including this one) in octets
			public UInt64 DataFieldLength;		//	8 integer, unsigned data field length, specifying the total size of the file data field in bits.
		}

		public class ImageStructureRecord
		{
			public Byte BitsPerPixel;			//	1 integer, unsigned number of bits per pixel (1 ... 255) NB
			public UInt16 Columns;				//	2 integer, unsigned number of columns (1 ... 65535) NC
			public UInt16 Lines;				//	2 integer, unsigned number of lines (1 ... 65535) NL
			public Byte Compression;			//	1 integer, unsigned compression flag (0,1,2) CFLG
		}

		public class ImageNavigationRecord
		{
			public String ProjectionName;		//	32 character projection name
			public Int32 ColumnScalingFactor;	//	4 integer, signed column scaling factor CFAC
			public Int32 LineScalingFactor;		//	4 integer, signed line scaling factor LFAC
			public Int32 ColumnOffset;			//	4 integer, signed column offset COFF
			public Int32 LineOffset;			//	4 integer, signed line offset LOFF
		}
		public static int[] ImageNavigationRecord_StringLengths = new int[] { 32 };
		
		public class AnnotationRecord
		{
			public String XRITchannelID;
			public String FieldSeparator0;
			public String DisseminationID;
			public String FieldSeparator1;
			public String DisseminatingSC;
			public String FieldSeparator2;
			public String ProductID1;
			public String FieldSeparator3;
			public String ProductID2;
			public String FieldSeparator4;
			public String ProductID3;
			public String FieldSeparator5;
			public String ProductID4;
			public String FieldSeparator6;
			public String Flags;

		}
		public static int[] AnnotationRecord_StringLengths = new int[] { 1, 1, 3, 1, 6, 1, 12, 1, 9, 1, 9, 1, 12, 1, 2 };

		public class TimeStampRecord
		{
			public Byte CDS_P_Field;
			public UInt16 CDS_T_Field_Days;			//	2 bytes counter of days starting from 1 January 1958
			public UInt32 CDS_T_Field_Milliseconds;	//	4 bytes milliseconds of day
		}

		public class SegmentIdentificationRecord
		{
			public UInt16 GP_SC_ID;
			public Byte Spectral_Channel_ID;
			public UInt16 Segm_Seq_No;
			public UInt16 Planned_Start_Segm_Seq_No;
			public UInt16 Planned_End_Segm_Seq_No;
			public Byte Representation;
		}



		//	FileTypeCode
		//	0			image data file
		//	1			GTS message
		//	2			alphanumeric text file
		//	3			encryption key message
		//	4...127		reserved for future global usage
		//	128...255	for mission specific use

		//	HeaderType
		//	0			primary header all -
		//	1			image structure 0 -
		//	2			image navigation - 0
		//	3			image data function - 0
		//	4			annotation - all
		//	5			time stamp - all
		//	6			ancillary text - all
		//	7			key header - all
		//	8...127		reserved for future global usage
		//	128...255	for mission specific use

		private static bool Endian = true;
		private static PixelFormat DefaultPixelFormat = PixelFormat.Format8bppIndexed;

		private static Spludlow.MetSat.DataItems DataItems = new DataItems();

		private static string[] fileTypesLookupData = new string[]
		{
				"0		Image Data",
				"1		GTS Message",
				"2		Alphanumeric Text",
				"128	Repeat Cycle Prologue",
				"129	Repeat Cycle Epilogue",
		};

		private static string[] headerTypesLookupData = new string[]   //	EPI & PRO only got: 4, 5
		{
				"1		Image Structure",
				"2		Image Navigation",
				"4		Annotation",
				"5		Time Stamp",
				"128	Image Segment Identification",	//	Segment Identification header will define the first and the last segment number of the image segments forming part of a repeat cycle
				"129	Image Segment Line Quality",
		};

		private static DataTable ChannelVaribles = Spludlow.Data.TextTable.ReadText(new string []
		{
			"DataProductId					Channel	Palette				Brightness",
			"String*						String*	String				Double",

			"EO_EUM_DAT_MSG_LRSEVIRI		IR_016	Blue-Green-2White	1.9",
			"EO_EUM_DAT_MSG_LRSEVIRI		IR_039	BlackRedYellowWhite	1.9",
			"EO_EUM_DAT_MSG_LRSEVIRI		IR_108	BlackRedYellowWhite	1.6",
			"EO_EUM_DAT_MSG_LRSEVIRI		VIS006	Blue-Green-2White	1.4",
			"EO_EUM_DAT_MSG_LRSEVIRI		WV_062	BlueCyan			1.3",

			"EO_EUM_DAT_MSG_HRSEVIRI		HRV		Blue-Green-4White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_016	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_039	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_087	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_097	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_108	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_120	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		IR_134	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		VIS006	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI		VIS008	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI		WV_062	BlueCyan			0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI		WV_073	GreenYellow			0.4",

			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	HRV		Blue-Green-4White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_016	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_039	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_087	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_097	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_108	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_120	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	IR_134	BlackRedYellowWhite	0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	VIS006	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	VIS008	Blue-Green-2White	0.4",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	WV_062	BlueCyan			0.3",
			"EO_EUM_DAT_MSG_HRSEVIRI-IODC	WV_073	GreenYellow			0.4",
		});

		public static void Report(string directory)
		{
			string[] columnLayouts = new string[]
			{
				"CId			0		1",
				"DId			2		3",
				"DSCId			6		6",
				"ProductID1		13		12",
				"ProductID2		26		9",			//	Channel
				"ProductID3		36		9",			//	File Sequance
				"ProductID4		46		12",		//	SensingTime
				"ext			58		3",
			};

			string[] filenames = Directory.GetFiles(directory);

			DataTable table = Spludlow.MetSat.Process.LoadFilenameInfo(columnLayouts, filenames);

			table.Columns.Add("SatIntDesId", typeof(string));
			table.Columns.Add("SensingTime", typeof(DateTime));
			table.Columns.Add("Channel", typeof(string));
			table.Columns.Add("Sequence", typeof(string));
			table.Columns.Add("Group", typeof(string));

			table.Columns.Add("FileTypeCode", typeof(byte));
			table.Columns.Add("DataFieldLength", typeof(ulong));

			table.Columns.Add("GP_SC_ID", typeof(UInt16));
			table.Columns.Add("Spectral_Channel_ID", typeof(Byte));
			table.Columns.Add("Segm_Seq_No", typeof(UInt16));
			table.Columns.Add("Planned_Start_Segm_Seq_No", typeof(UInt16));
			table.Columns.Add("Planned_End_Segm_Seq_No", typeof(UInt16));

			table.Columns.Add("NavigationColumnOffset", typeof(int));
			table.Columns.Add("NavigationLineOffset", typeof(int));

			table.Columns.Add("StructureBitsPerPixel", typeof(byte));
			table.Columns.Add("StructureColumns", typeof(ushort));
			table.Columns.Add("StructureLines", typeof(ushort));
						
			table.Columns.Add("HeaderTypes", typeof(string));

			List<string> groups = new List<string>();

			foreach (DataRow row in table.Rows)
			{
				string satId = (string)row["ProductID1"];
				if (satId.StartsWith("MPEF_") == true)
					satId = (string)row["DSCId"];

				string satIntDesId = satId;
				string senseTime = (string)row["ProductID4"];

				string group = satIntDesId + "-" + senseTime;

				row["SatIntDesId"] = satIntDesId;
				row["SensingTime"] = DateTime.ParseExact(senseTime, "yyyyMMddHHmm", null);
				row["Channel"] = ((string)row["ProductID2"]).Trim(new char[] { '_' });
				row["Sequence"] = ((string)row["ProductID3"]).Trim(new char[] { '_' });

				row["Group"] = group;

				if (groups.Contains(group) == false)
					groups.Add(group);



			}

			groups.Sort();

			List<DataTable> groupTables = new List<DataTable>();

			for (int groupIndex = 0; groupIndex < groups.Count; ++groupIndex)
			{
				string group = groups[groupIndex];

				DataTable groupTable = table.Clone();
				foreach (DataRow row in table.Select("Group = '" + group + "'", "Channel, Sequence"))
					groupTable.ImportRow(row);

				groupTables.Add(groupTable);

				foreach (DataRow row in groupTable.Rows)
				{
					string filename = (string)row["Filename"];

					XritFile xritFile = LoadFile(filename);

					row["FileTypeCode"] = xritFile.Header.FileTypeCode;
					row["DataFieldLength"] = xritFile.Header.DataFieldLength;

					if (xritFile.SegmentIdentification != null)
					{
						row["GP_SC_ID"] = xritFile.SegmentIdentification.GP_SC_ID;
						row["Spectral_Channel_ID"] = xritFile.SegmentIdentification.Spectral_Channel_ID;
						row["Segm_Seq_No"] = xritFile.SegmentIdentification.Segm_Seq_No;
						row["Planned_Start_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_Start_Segm_Seq_No;
						row["Planned_End_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_End_Segm_Seq_No;
					}

					if (xritFile.Navigation != null)
					{
						row["NavigationColumnOffset"] = xritFile.Navigation.ColumnOffset;
						row["NavigationLineOffset"] = xritFile.Navigation.LineOffset;
					}

					if (xritFile.Structure != null)
					{
						row["StructureBitsPerPixel"] = xritFile.Structure.BitsPerPixel;
						row["StructureColumns"] = xritFile.Structure.Columns;
						row["StructureLines"] = xritFile.Structure.Lines;
					}
					

					StringBuilder headerTypes = new StringBuilder();
					foreach (byte headerType in xritFile.HeaderTypes)
					{
						if (headerTypes.Length > 0)
							headerTypes.Append(", ");
						headerTypes.Append(headerType);
					}
					row["HeaderTypes"] = headerTypes.ToString();


				}
				
				bool groupComlete = IsComplete(groupTable);

				//	If not complete and on last group then skip
				if (groupComlete == false && (groupIndex == (groups.Count - 1)))
				{
					Spludlow.Log.Warning("XRIT Last Group Incomplete skipping: " + group);
					continue;
				}
					

				Spludlow.Log.Info("Complete " + group + " " + groupComlete.ToString());
			}

			Spludlow.Log.Report("Report", groups.ToArray(), groupTables.ToArray());	//, table);
		}

		public static bool IsComplete(DataTable groupTable)
		{
			DataRow[] rows;

			rows = groupTable.Select("FileTypeCode = 129");

			if (rows.Length == 1)
				return true;
			if (rows.Length > 1)
				throw new ApplicationException("XRIT IsComplete Group table has more than one EPI(129)");
			
			List<int> channels = new List<int>();

			foreach (DataRow row in groupTable.Rows)
			{
				if (row.IsNull("Spectral_Channel_ID") == true)
					continue;

				int channel = (byte)row["Spectral_Channel_ID"];

				if (channels.Contains(channel) == false)
					channels.Add(channel);
			}

			if (channels.Count == 0)
				return false;

			foreach (int channel in channels)
			{
				rows = groupTable.Select("Spectral_Channel_ID = " + channel);

				int last = (UInt16)rows[0]["Planned_End_Segm_Seq_No"];

				if (rows.Length != last)
					return false;
			}

			return true;
		}

		public static void Run(string directory)
		{
			string dataProductId = Path.GetFileName(directory).ToUpper();

			bool grib = false;

			if (dataProductId == "EO_EUM_DAT_MSG_CLM") 
				 grib = true;
			if (dataProductId == "EO_EUM_DAT_MSG_CLM-IODC")
				grib = true;

			string[] columnLayouts = new string[]
			{
				"CId			0		1",
				"DId			2		3",	
				"DSCId			6		6",	
				"ProductID1		13		12",
				"ProductID2		26		9",			//	Channel
				"ProductID3		36		9",			//	File Sequance
				"ProductID4		46		12",		//	SensingTime
				"ext			58		3",
			};

			string[] filenames = Directory.GetFiles(directory);

			DataTable table = Spludlow.MetSat.Process.LoadFilenameInfo(columnLayouts, filenames);

			table.Columns.Add("SatIntDesId", typeof(string));
			table.Columns.Add("SensingTime", typeof(DateTime));
			table.Columns.Add("Channel", typeof(string));
			table.Columns.Add("Sequence", typeof(string));
			table.Columns.Add("Group", typeof(string));

			table.Columns.Add("FileTypeCode", typeof(byte));
			table.Columns.Add("DataFieldLength", typeof(ulong));

			table.Columns.Add("GP_SC_ID", typeof(UInt16));
			table.Columns.Add("Spectral_Channel_ID", typeof(Byte));
			table.Columns.Add("Segm_Seq_No", typeof(UInt16));
			table.Columns.Add("Planned_Start_Segm_Seq_No", typeof(UInt16));
			table.Columns.Add("Planned_End_Segm_Seq_No", typeof(UInt16));

			table.Columns.Add("NavigationColumnOffset", typeof(int));
			table.Columns.Add("NavigationLineOffset", typeof(int));

			table.Columns.Add("StructureBitsPerPixel", typeof(byte));
			table.Columns.Add("StructureColumns", typeof(ushort));
			table.Columns.Add("StructureLines", typeof(ushort));

			table.Columns.Add("HeaderTypes", typeof(string));

			List<string> groups = new List<string>();

			foreach (DataRow row in table.Rows)
			{
				string satId = (string)row["ProductID1"];
				if (satId.StartsWith("MPEF_") == true)
					satId = (string)row["DSCId"];

				string satIntDesId = satId;
				string senseTime = (string)row["ProductID4"];

				string group = satIntDesId + "-" + senseTime;

				row["SatIntDesId"] = satIntDesId;
				row["SensingTime"] = DateTime.ParseExact(senseTime, "yyyyMMddHHmm", null);
				row["Channel"] = ((string)row["ProductID2"]).Trim(new char[] { '_' });
				row["Sequence"] = ((string)row["ProductID3"]).Trim(new char[] { '_' });

				row["Group"] = group;

				if (groups.Contains(group) == false)
					groups.Add(group);
			}

			groups.Sort();

			Spludlow.MetSat.DataItems dataItems = new Spludlow.MetSat.DataItems();

			for (int groupIndex = 0; groupIndex < groups.Count; ++groupIndex)
			{
				string group = groups[groupIndex];

				DataTable groupTable = table.Clone();
				foreach (DataRow row in table.Select("Group = '" + group + "'", "Channel, Sequence"))
					groupTable.ImportRow(row);

				Dictionary<string, XritFile> xritFiles = new Dictionary<string, XritFile>();

				foreach (DataRow row in groupTable.Rows)
				{
					string filename = (string)row["Filename"];

					XritFile xritFile = LoadFile(filename);

					xritFiles.Add(filename, xritFile);

					row["FileTypeCode"] = xritFile.Header.FileTypeCode;
					row["DataFieldLength"] = xritFile.Header.DataFieldLength;

					if (xritFile.SegmentIdentification != null)
					{
						row["GP_SC_ID"] = xritFile.SegmentIdentification.GP_SC_ID;
						row["Spectral_Channel_ID"] = xritFile.SegmentIdentification.Spectral_Channel_ID;
						row["Segm_Seq_No"] = xritFile.SegmentIdentification.Segm_Seq_No;
						row["Planned_Start_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_Start_Segm_Seq_No;
						row["Planned_End_Segm_Seq_No"] = xritFile.SegmentIdentification.Planned_End_Segm_Seq_No;
					}

					if (xritFile.Navigation != null)
					{
						row["NavigationColumnOffset"] = xritFile.Navigation.ColumnOffset;
						row["NavigationLineOffset"] = xritFile.Navigation.LineOffset;
					}

					if (xritFile.Structure != null)
					{
						row["StructureBitsPerPixel"] = xritFile.Structure.BitsPerPixel;
						row["StructureColumns"] = xritFile.Structure.Columns;
						row["StructureLines"] = xritFile.Structure.Lines;
					}


					StringBuilder headerTypes = new StringBuilder();
					foreach (byte headerType in xritFile.HeaderTypes)
					{
						if (headerTypes.Length > 0)
							headerTypes.Append(", ");
						headerTypes.Append(headerType);
					}
					row["HeaderTypes"] = headerTypes.ToString();


				}

				bool groupComlete = IsComplete(groupTable);

				//	If not complete and on last group then skip
				if (groupComlete == false && (groupIndex == (groups.Count - 1)))
				{
					Spludlow.Log.Warning("XRIT Group Incomplete skipping: " + dataProductId + " " + directory + ", " + group, groupTable);
					continue;
				}

				if (grib == false)
					LoadGroup(groupTable, dataProductId, dataItems, xritFiles);
				else
					LoadGribGroup(groupTable, dataProductId, dataItems, xritFiles);

				Spludlow.Log.Report(dataProductId + " " + directory + ", " + group, groupTable);

			}

			Spludlow.Log.Finish(dataProductId + " Finished; Groups:" + groups.Count + ", Files:" + table.Rows.Count);
		}

		private static void LoadGribGroup(DataTable groupTable, string dataProductId, Spludlow.MetSat.DataItems dataItems, Dictionary<string, XritFile> xritFiles)
		{
			List<XritFile> xrits = new List<XritFile>();

			DataRow firstRow = null;

			int dataSize = 0;

			List<string> sourceFilenames = new List<string>();

			foreach (DataRow row in groupTable.Rows)
			{
				string filename = (string)row["Filename"];
				string sequenceText = (string)row["Sequence"];

				try
				{
					int sequence = 0;

					if (sequenceText[0] == '0')
						sequence = Int32.Parse(sequenceText);

					if (sequence > 0)
					{
						if (firstRow == null)
							firstRow = row;

						XritFile info = xritFiles[filename];
						xrits.Add(info);

						dataSize += info.Data.Length;
					}
				}
				catch (Exception ee)
				{
					throw new ApplicationException("Loading File :\t" + filename + "\t" + ee.Message, ee);
				}

				sourceFilenames.Add(filename);
			}

			if (firstRow != null)
			{
				Schema.DataItemFilesDataTable filesTable = new Schema.DataItemFilesDataTable();

				filesTable.DataItemFileIdColumn.ReadOnly = false;
				filesTable.DataItemIdColumn.AllowDBNull = true;

				string satIntDesId = (string)firstRow["SatIntDesId"];
				DateTime sensingTime = (DateTime)firstRow["SensingTime"];

				string dataGroupName = "SatId" + " - " + "SEVIRI" + " - " + Spludlow.Text.TimeStamp(sensingTime);

				using (Spludlow.TempDirectory tempDir = new TempDirectory())
				{
					Schema.DataItemFilesRow fileRow;

					//	Source Archive

					string name = dataGroupName + "-SourceArchive";

					string tempCopyDirectory = tempDir + @"\" + name;

					Directory.CreateDirectory(tempCopyDirectory);

					foreach (string sourceFilename in sourceFilenames)
						File.Copy(sourceFilename, tempCopyDirectory + @"\" + Path.GetFileName(sourceFilename));

					fileRow = dataItems.MakeFileRow(filesTable, "SA", name + ".7z", "");

					string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });

					Archive.Create(tempStoreName[0], tempCopyDirectory, true);

					// Bitmap

					string channel = "CLM"; //	!!!!!!!!!!!!!!!!!!!!!! may be others not supported yet

					name = dataGroupName + "-" + channel;

					byte[] gribData = new byte[dataSize];
					int offset = 0;
					foreach (XritFile xrit in xrits)
					{
						Buffer.BlockCopy(xrit.Data, 0, gribData, offset, xrit.Data.Length);
						offset += xrit.Data.Length;
					}

					try
					{
						Spludlow.MetSat.GRIB grib = new GRIB(gribData);

						fileRow = dataItems.MakeFileRow(filesTable, "CH", name + "-" + channel + ".png", channel);

						tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });

						Spludlow.MetSat.CLM.ProcessGRIB(grib, tempStoreName[0]);

						Size size = Spludlow.Drawing.Bitmaps.Dimensions(tempStoreName[0]);

						fileRow.Size = (int)Spludlow.Io.Files.FileLength(tempStoreName[0]);
						fileRow.Width = size.Width;
						fileRow.Height = size.Height;
						fileRow.Pixels = size.Width * size.Height;
					}
					catch (Exception ee)
					{
						Spludlow.Log.Warning("MetSat XRIT Processing GRIB data: " + name, ee);
					}

					//	Save

					dataItems.SaveDataItem(filesTable, satIntDesId, dataProductId, sensingTime, dataGroupName);
				}
			}

			foreach (string sourceFilename in sourceFilenames)
				File.Delete(sourceFilename);
		}

		private static void LoadGroup(DataTable groupTable, string dataProductId, Spludlow.MetSat.DataItems dataItems, Dictionary<string, XritFile> xritFiles)
		{
			Dictionary<string, List<XritFile>> channelGroupData = new Dictionary<string, List<XritFile>>();

			DataRow firstRow = null;

			List<string> sourceFilenames = new List<string>();

			foreach (DataRow row in groupTable.Rows)
			{
				string filename = (string)row["Filename"];
				string channel = (string)row["Channel"];
				string sequenceText = (string)row["Sequence"];

				try
				{
					int sequence = 0;

					if (sequenceText[0] == '0')
						sequence = Int32.Parse(sequenceText);

					if (sequence > 0)
					{
						if (firstRow == null)
							firstRow = row;

						XritFile info = xritFiles[filename];

						if (channelGroupData.ContainsKey(channel) == false)
							channelGroupData[channel] = new List<XritFile>();

						channelGroupData[channel].Add(info);
					}
				}
				catch (Exception ee)
				{
					throw new ApplicationException("Loading File :\t" + filename + "\t" + ee.Message, ee);
				}

				sourceFilenames.Add(filename);
			}
			
			if (firstRow != null)
			{
				Schema.DataItemFilesDataTable filesTable = new Schema.DataItemFilesDataTable();

				filesTable.DataItemFileIdColumn.ReadOnly = false;
				filesTable.DataItemIdColumn.AllowDBNull = true;
				
				string satIntDesId = (string)firstRow["SatIntDesId"];
				//string filename = Path.GetFileName((string)firstRow["Filename"]);
				DateTime sensingTime = (DateTime)firstRow["SensingTime"];

				//	Spludlow.MetSat.Satellites.SatelliteName(satIntDesId)

				string dataGroupName = "SatId" + " - " + "SEVIRI" + " - " + Spludlow.Text.TimeStamp(sensingTime);

				using (Spludlow.TempDirectory tempDir = new TempDirectory())
				{
					Schema.DataItemFilesRow fileRow;

					//	Source Archive

					string name = dataGroupName + "-SourceArchive";

					string tempCopyDirectory = tempDir + @"\" + name;

					Directory.CreateDirectory(tempCopyDirectory);

					foreach (string sourceFilename in sourceFilenames)
						File.Copy(sourceFilename, tempCopyDirectory + @"\" + Path.GetFileName(sourceFilename));

					fileRow = dataItems.MakeFileRow(filesTable, "SA", name + ".7z", "");

					string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });

					Archive.Create(tempStoreName[0], tempCopyDirectory, true);


					//Spludlow.Log.Report(dataGroupName, groupTable);


					// Channel Bitmpas

					OutputChannelBitmaps(channelGroupData, dataGroupName + "-Channel", dataItems, filesTable, dataProductId);

					//	Save
					
					dataItems.SaveDataItem(filesTable, satIntDesId, dataProductId, sensingTime, dataGroupName);
				}
			}

			foreach (string sourceFilename in sourceFilenames)
				File.Delete(sourceFilename);
		}


		public static void Draw(XritFile xritFile, Bitmap bitmap, int yOffset, double scale)
		{
			byte[] bitmapData = Spludlow.Drawing.Bitmaps.ReadBitmapData(bitmap, DefaultPixelFormat);

			int[] values = Values(xritFile);

			for (int y = 0; y < xritFile.Structure.Lines; ++y)
			{
				for (int x = 0; x < xritFile.Structure.Columns; ++x)
				{
					int index = (y * xritFile.Structure.Columns) + x;

					int targetSegmentY = (xritFile.Structure.Lines - 1) - y;
					int targetX = ((xritFile.Structure.Columns - 1) - x);

					int destinationIndex = ((targetSegmentY + yOffset) * xritFile.Structure.Columns) + targetX;

					int value = values[index];

					if (scale != 1)
					{
						value = (int)Math.Round((double)value * scale, 0);

						if (value > 0xFF)
							value = 0xFF;
					}

					bitmapData[destinationIndex] = (byte)value;
				}
			}

			Spludlow.Drawing.Bitmaps.WriteBitmapData(bitmap, bitmapData, DefaultPixelFormat);
		}

		private static void OutputChannelBitmaps(Dictionary<string, List<XritFile>> channelGroupData, string name, Spludlow.MetSat.DataItems dataItems, Schema.DataItemFilesDataTable filesTable, string dataProductId)
		{
			int highShadeMaxCount = 100;

			Spludlow.Drawing.ACT act = new Drawing.ACT();
			
			foreach (string channel in channelGroupData.Keys)
			{
				string palette = "grey";
				double brightness = 1.0;
				
				DataRow channelRow = ChannelVaribles.Rows.Find(new object[] { dataProductId, channel });
				if (channelRow != null)
				{
					palette = (string)channelRow["Palette"];
					brightness = (double)channelRow["Brightness"];
				}
				else
				{
					Spludlow.Log.Warning("XRIT; OutputChannelBitmaps, ChannelVaribles row not found:\t" + dataProductId + ", " + channel);		//	Alow not present for deafults
				}
				
				Dictionary<int, int> histogram = new Dictionary<int, int>();

				foreach (XritFile info in channelGroupData[channel])
				{
					if (brightness == 0 && info.Header.FileTypeCode == 0)
					{
						int[] values = Values(info);		// when using hostogram regetting values below !!!!!!!!!

						foreach (int value in values)
						{
							if (histogram.ContainsKey(value) == false)
								histogram.Add(value, 1);
							else
								histogram[value] = histogram[value] + 1;
						}
					}
				}

				if (brightness == 0)
				{
					DataTable histogramTable = Spludlow.Data.TextTable.ReadText(new string[]
					{
						"Value		Count",
						"Int32*		Int32",
					});

					foreach (int value in histogram.Keys)
						histogramTable.Rows.Add(new object[] { value, histogram[value] });

					DataView view = new DataView(histogramTable);
					view.Sort = "Value";

					int high = (int)view[view.Count -1].Row["Value"];

					for (int viewIndex = view.Count - 1; viewIndex > 0 && (int)view[viewIndex].Row["Count"] < highShadeMaxCount; --viewIndex)
					{
						high = (int)view[viewIndex].Row["Value"];
					}

					double scale = (double)0xFF / (double)high;
					
					Spludlow.Log.Report("XRIT Histogram;\t" + name + "-" + channel + ", High:" + high + ", Scale:" + scale, new object[] { view });

					brightness = scale;
				}

				XritFile xritFirst = channelGroupData[channel][0];

				int imageWidth = xritFirst.Structure.Columns;
				int imageHeight = xritFirst.Structure.Lines * xritFirst.SegmentIdentification.Planned_End_Segm_Seq_No;
				int segmentHeight = xritFirst.Structure.Lines;

				using (Bitmap bitmap = new Bitmap(imageWidth, imageHeight, DefaultPixelFormat))
				{
					if (palette == "grey")
					{
						Spludlow.Drawing.Bitmaps.SetPalette(bitmap);
					}
					else
					{
						byte[] paletteData = act.Get(palette);
						paletteData[0] = 0;
						paletteData[1] = 0;
						paletteData[2] = 0;
						Spludlow.Drawing.Bitmaps.SetPalette(bitmap, paletteData);
					}
					
					int moveDown = 0;       //	For some reason the HRV channel offsets are out by -2 ???
					if (imageWidth != imageHeight)
						moveDown = 2;

					foreach (XritFile xritFile in channelGroupData[channel])
					{
						int yOffset = xritFile.Navigation.LineOffset + moveDown + (imageHeight / 2) - segmentHeight;

						Draw(xritFile, bitmap, yOffset, brightness);
					}
				
					Schema.DataItemFilesRow fileRow = dataItems.MakeFileRow(filesTable, "CH", name + "-" + channel + ".png", channel);

					string[] tempStoreName = ((string)fileRow["Filename"]).Split(new char[] { '@' });

					bitmap.Save(tempStoreName[0], ImageFormat.Png);

					fileRow.Size = (int)Spludlow.Io.Files.FileLength(tempStoreName[0]);
					fileRow.Width = imageWidth;
					fileRow.Height = imageHeight;
					fileRow.Pixels = imageWidth * imageHeight;
				}
			}
		}

		public static XritFile LoadFile(string filename)
		{
			XritFile xritFile = new XritFile();
			xritFile.Header = new PrimaryHeaderRecord();

			using (FileStream stream = new FileStream(filename, FileMode.Open))
			{
				using (BinaryReader reader = new BinaryReader(stream))
				{
					long read;

					read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Header, Endian);

					long headerLeft = xritFile.Header.TotalHeaderLength - read;

					List<byte> headerTypeList = new List<byte>();

					while (headerLeft > 0)
					{
						byte headerType = reader.ReadByte();
						UInt16 headerRecordLength = Spludlow.Io.BinaryIO.ReadUInt16(reader, Endian);

						headerTypeList.Add(headerType);

						switch (headerType)
						{
							case 1:
								xritFile.Structure = new ImageStructureRecord();
								read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Structure, Endian);
								break;

							case 2:
								xritFile.Navigation = new ImageNavigationRecord();
								read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Navigation, Endian, ImageNavigationRecord_StringLengths);
								break;

							case 4:
								xritFile.Annotation = new AnnotationRecord(); 
								read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.Annotation, Endian, AnnotationRecord_StringLengths);
								break;

							case 5:
								xritFile.TimeStamp = new TimeStampRecord();
								read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.TimeStamp, Endian);
								break;

							case 128:
								xritFile.SegmentIdentification = new SegmentIdentificationRecord();
								read = Spludlow.Io.BinaryIO.ReadBinary(reader, xritFile.SegmentIdentification, Endian);
								break;

							default:
								read = headerRecordLength - 3;
								reader.ReadBytes((int)read);
								break;
						}

						if (read != (headerRecordLength - 3))
							throw new ApplicationException("XRIT.LoadFile; Bad Header Size:\t" + filename);

						headerLeft -= headerRecordLength;
					}

					if (headerLeft != 0)
						throw new ApplicationException("XRIT.LoadFile; Bad Header Over Read:\t" + filename);

					int dataSizeBytes = (int)(xritFile.Header.DataFieldLength / 8);
					if ((xritFile.Header.DataFieldLength % 8) > 0)
						++dataSizeBytes;

					xritFile.Data = reader.ReadBytes(dataSizeBytes);

					if (reader.BaseStream.Position != reader.BaseStream.Length)
						throw new ApplicationException("XRIT.LoadFile; More data than expected past end of file:\t" + filename);

					xritFile.HeaderTypes = headerTypeList.ToArray();
				}
			}

			return xritFile;
		}

		public static DataTable Histogram(string[] filenames)
		{
			DataTable table = Spludlow.Data.TextTable.ReadText(new string[]
			{
				"Value		Count",
				"Int32*		Int32",
			});

			foreach (string filename in filenames)
			{
				XritFile xritFile = LoadFile(filename);
				
				if (xritFile.Header.FileTypeCode != 0)
					continue;

				foreach (int value in Values(xritFile))
				{
					DataRow row = table.Rows.Find(value);
					if (row == null)
						row = table.Rows.Add(new object[] { value, 0 });

					row["Count"] = (int)row["Count"] + 1;
				}
			}

			return table;
		}

		public static int[] Values(XritFile xritFile)	//, double scale)		//	have scale only on other mathds . others will use values. dont re get for hostorgam
		{
			int total = xritFile.Structure.Columns * xritFile.Structure.Lines;

			int[] values = new int[total];

			for (int index = 0; index < total; ++index)
			{
				if (xritFile.Structure.BitsPerPixel == 8)
				{
					values[index] = xritFile.Data[index];
				}
				else
				{
					int bitIndex = index * xritFile.Structure.BitsPerPixel;

					byte[] bytes = ExtractBytes(bitIndex, xritFile.Data);

					int value = Spludlow.Io.BinaryIO.ReadUInt16(bytes, 0, Endian);

					values[index] = value;
				}
			}

			return values;
		}

		public static void Write(XritFile xritFile, string filename, int dpi)
		{
			using (Bitmap bitmap = new Bitmap(xritFile.Structure.Columns, xritFile.Structure.Lines, DefaultPixelFormat))
			{
				bitmap.SetResolution(dpi, dpi);
				Spludlow.Drawing.Bitmaps.SetPalette(bitmap);

				Draw(xritFile, bitmap, 0, 1);

				bitmap.Save(filename, ImageFormat.Png);
			}
		}

		public static byte[] ExtractBytes(int bitIndex, byte[] imageData)
		{
			//	The bits of one pixel are oriented from MSB through LSB

			//	Only tested 10 bit

			int byteIndex = bitIndex / 8;
			int bitOffset = bitIndex % 8;

			byte[] bytes = new byte[2];
			bytes[0] = imageData[byteIndex];
			bytes[1] = imageData[byteIndex + 1];

			int shift = 6 - bitOffset;

			if (bitOffset > 0)
			{
				//	Blank out bits before offset in 1st byte
				int mask = (int)Math.Pow(2, shift + 2) - 1;
				bytes[0] = (byte)(bytes[0] & mask);
			}

			if (bytes[0] == 0 && bytes[1] == 0)
				return bytes;

			if (shift > 0)
			{
				//	move along bits in 2nd byte
				bytes[1] = (byte)(bytes[1] >> shift);

				//	copy bits from end of 1st byte into start of 2nd byte
				int mask = (int)Math.Pow(2, shift) - 1;
				byte temp = (byte)((bytes[0] & mask) << (bitOffset + 2));
				bytes[1] = (byte)(bytes[1] | temp);

				//	move along start of 1st byte
				bytes[0] = (byte)(bytes[0] >> shift);
			}

			return bytes;
		}

	}
}