//	Spludlow Software
//	Copyright © Samuel P. Ludlow 2020 All Rights Reserved
//	Distributed under the terms of the GNU General Public License version 3
//	Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE
//	https://www.spludlow.co.uk/LICENCE.TXT
//	The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission
//	v1.14

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using System.IO;
using System.Data;

namespace Spludlow.Data
{
	/// <summary>
	/// Some Methods for performing various MAME ROM Related tasks
	/// 
	/// Put the official XML into a DB the "machine" and "rom" tables are used here
	/// 
	/// Index & Import (2 Steps) ROM collections into a "SHA1 Store". All ROMS are hashed and stored with the SHA1 as the filename
	/// 
	/// Create machine ROM set zip files from the DB and SHA1 store.
	/// 
	/// Create a .bat file to run through specified games (so you can look for the good ones)
	/// 
	/// !!! use the temp path with syntax \\?\C:\TEMP to overcome long filname problems
	/// 
	/// TempDirectory & Ram DISK
	/// ========================
	/// If TempDirectory is null the C: drive will be used
	/// Used to extract/create archives.
	/// Using a RAM disk or seperate fast drive may improve performance.
	/// Using 4.5GB seems to cover it.
	/// You can run on a smaller RAM disk then run a 2nd pass without it to get the larger stuff
	/// Any failures indexing (including disk full) will go into ".bad.txt"
	/// </summary>
	public class Mame
	{
		/// <summary>
		/// Convert MAME full driver information XML into releational database
		/// </summary>
		public static void RelationizeXmlToDatabase(string xmlFilename, string connectionString, string databaseName, string tempDirectory)
		{
			string topElementname = "machine";

			DataSet dataSet = Spludlow.Data.XML.ConvertRelational(xmlFilename, topElementname);

			DataSet schema = Spludlow.Data.ADO.Schema(dataSet, true);

			Spludlow.Data.Database.AutoAddForeignKeys(schema);

			Spludlow.Log.Report("MAME Relationize Xml Schema: " + topElementname, schema);

			Spludlow.Data.Schemas.ImportDatabase(schema, dataSet, connectionString, databaseName, tempDirectory);
		}


		/// <summary>
		/// Some rom collections have directories and not zips, fix with this method
		/// </summary>
		public static void ZipRomDirectory(string directory)
		{
			foreach (string romDirectory in Directory.GetDirectories(directory))
			{
				string archiveName = romDirectory + ".zip";

				if (File.Exists(archiveName) == true)
					continue;

				Spludlow.Archive.Create(archiveName, romDirectory + @"\*");

				Directory.Delete(romDirectory, true);
			}
		}

		public static void FullImportSoftwareDirectory(string softwareRootDirectory, string storeDirectory, string tempDirectory)
		{
			IndexSoftwareDirectory(softwareRootDirectory, tempDirectory);

			foreach (string directory in Directory.GetDirectories(softwareRootDirectory))
			{
				string indexFilename = directory + ".txt";

				if (File.Exists(indexFilename) == true)
				{
					LoadRoms(indexFilename, storeDirectory, tempDirectory);

					DeleteImportedRoms(directory);
				}
			}
		}

		/// <summary>
		/// 
		/// </summary>
		public static void IndexSoftwareDirectory(string directory, string tempDirectory)
		{
			foreach (string listDirectory in Directory.GetDirectories(directory))
			{
				string softwareListName = Path.GetFileName(listDirectory);

				string[] softwareFilenames = Directory.GetFiles(listDirectory);
				string[] softwareSubDirs = Directory.GetDirectories(listDirectory);
				int zipCount = 0;
				int nonZipCount = 0;
				foreach (string filename in softwareFilenames)
				{
					if (Path.GetExtension(filename).ToLower() == ".zip")
						++zipCount;
					else
						++nonZipCount;
				}

				bool error = false;
				try
				{
					IndexRomDirectory(listDirectory, tempDirectory);
				}
				catch (Exception ee)
				{
					Spludlow.Log.Warning("IndexSoftwareDirectory: " + directory, ee);
					error = true;
				}

				if (error == true || zipCount == 0 || nonZipCount > 0 || softwareSubDirs.Length > 0)
					File.AppendAllText(listDirectory + ".bad.txt", listDirectory + Environment.NewLine);
			}
			
		}

		public static void ReportSoftwareListHashes(string hashStoreDirectory)
		{
			HashSet<string> storeHashes = ReadExistingHashes(hashStoreDirectory);

			Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create("SPLCAL-MAIN@MameSoftware");

			HashSet<string> databaseHashes = new HashSet<string>();

			foreach (string tableName in new string[] { "disk", "rom" })
			{
				foreach (DataRow row in database.Select("SELECT sha1 FROM " + tableName).Rows)
				{
					if (row.IsNull("sha1") == true)
						continue;
					string sha1 = ((string)row["sha1"]).ToUpper();
					if (databaseHashes.Contains(sha1) == false)
						databaseHashes.Add(sha1);
				}
			}

			ReportDiff(databaseHashes, storeHashes, "Database", "FileStore", false);


		}

		public static void LoadSoftware(string directory, string storeDirectory, string tempDirectory)
		{
			//int count = 10;

			foreach (string listDirectory in Directory.GetDirectories(directory))
			{
				string softwareListName = Path.GetFileName(listDirectory);

				string indexName = listDirectory + ".txt";

				LoadRoms(indexName, storeDirectory, tempDirectory);

				Spludlow.Log.Info("MAME LoadSoftware: " + listDirectory);

				//if (++count == 10)
				//	break;
			}

			Spludlow.Log.Finish("MAME LoadSoftware");
		}

		public static void TestExtract(string zipFilename, string tempDirectory)
		{
			string parentMachineName = Path.GetFileNameWithoutExtension(zipFilename);
			string archiveDirectory;

			archiveDirectory = tempDirectory + @"\" + parentMachineName;
			Directory.CreateDirectory(archiveDirectory);

			Spludlow.Archive.Extract(zipFilename, archiveDirectory);

			Directory.Delete(archiveDirectory, true);

			//using (Spludlow.TempDirectory tempDir = new Spludlow.TempDirectory(tempDirectory))
			//{
			//	archiveDirectory = tempDir.Path + @"\" + parentMachineName;
			//	Directory.CreateDirectory(archiveDirectory);

			//	Spludlow.Archive.Extract(zipFilename, archiveDirectory);

			//	//foreach (string filename in Directory.GetFiles(archiveDirectory, "*", SearchOption.AllDirectories))
			//	//	File.SetAttributes(filename, FileAttributes.Normal);
			//}
		}

		/// <summary>
		/// STEP 1
		/// Create a text table index for the contents all rom zips, result filename is directory + ".txt"
		/// Use null for the temp dir or a RAM disk for speed
		/// </summary>
		public static void IndexRomDirectory(string directory, string tempDirectory)
		{
			DataTable resultsTable = Spludlow.Data.TextTable.ReadText(new string[]
			{
				"RomFileId	ArchiveName	MachineName	RomName	SHA1	Length",
				"Int32		String		String		String	String	Int64",
			});

			int id = 0;

			List<string> badList = new List<string>();


			foreach (string zipFilename in Directory.GetFiles(directory))
			{
				string parentMachineName = Path.GetFileNameWithoutExtension(zipFilename);
				string archiveDirectory;

				using (Spludlow.TempDirectory tempDir = new Spludlow.TempDirectory(tempDirectory))
				{
					List<string> romFilenames = new List<string>();

					try
					{
						archiveDirectory = tempDir.Path + @"\" + parentMachineName;
						Directory.CreateDirectory(archiveDirectory);

						Spludlow.Archive.Extract(zipFilename, archiveDirectory);

						foreach (string filename in Directory.GetFiles(archiveDirectory, "*", SearchOption.AllDirectories))
							File.SetAttributes(filename, FileAttributes.Normal);

						string[] filenames;
						string[] subDirs;
						
						filenames = Directory.GetFiles(archiveDirectory);

						if (filenames.Length == 0)
							throw new ApplicationException("No filenames in archive root: " + zipFilename);

						foreach (string filename in filenames)
							romFilenames.Add(filename);

						subDirs = Directory.GetDirectories(archiveDirectory);

						foreach (string subDirectory in subDirs)
						{
							string childMachine = Path.GetFileName(subDirectory);
							if (childMachine == parentMachineName)
								throw new ApplicationException("Parent name in child: " + zipFilename);

							if (Directory.GetDirectories(subDirectory).Length > 0)
								throw new ApplicationException("Sub directories in child: " + zipFilename);

							filenames = Directory.GetFiles(subDirectory);

							if (filenames.Length == 0)
								throw new ApplicationException("No filenames in child: " + zipFilename);

							foreach (string filename in filenames)
								romFilenames.Add(filename);
						}
					}
					catch
					{
						badList.Add(zipFilename);
						continue;
					}

					foreach (string filename in romFilenames)
					{
						string machineName = Path.GetFileName(Path.GetDirectoryName(filename));
						string romName = filename.Substring(archiveDirectory.Length + 1);

						string sha1 = Spludlow.Hashing.SHA1HexFile(filename);

						long length = Spludlow.Io.Files.FileLength(filename);

						resultsTable.Rows.Add(++id, parentMachineName, machineName, romName, sha1, length);
					}
				}
			}

			Spludlow.Data.TextTable.Write(directory + ".txt", resultsTable, Encoding.UTF8);

			if (badList.Count > 0)
			{
				File.WriteAllLines(directory + ".bad.txt", badList.ToArray());
			}

		}

		/// <summary>
		/// STEP 2
		/// Import roms that the SHA1 store does not already contain
		/// Using the Index from STEP 1 only the zips required are extracted
		/// Use null for the temp dir or a RAM disk for speed
		/// </summary>
		public static void LoadRoms(string romIndexFilename, string storeDirectory, string tempDirectory)
		{
			string sourceDirectory = romIndexFilename.Substring(0, romIndexFilename.Length - 4);

			DataTable table = Spludlow.Data.TextTable.ReadFile(romIndexFilename, Encoding.UTF8);

			HashSet<string> hashes = ReadExistingHashes(storeDirectory);

			HashSet<string> wantedArchives = new HashSet<string>();

			foreach (DataRow row in table.Rows)
			{
				string sha1 = (string)row["SHA1"];

				if (hashes.Contains(sha1) == true)
					continue;

				string archive = (string)row["ArchiveName"];

				if (wantedArchives.Contains(archive) == false)
					wantedArchives.Add(archive);
			}

			int startCount = hashes.Count;

			foreach (string archive in wantedArchives)
			{
				string sourceZipFilename = sourceDirectory + @"\" + archive + ".zip";

				if (File.Exists(sourceZipFilename) == false)
					throw new ApplicationException("sourceZipFilename: " + sourceZipFilename);

				using (Spludlow.TempDirectory tempDir = new Spludlow.TempDirectory(tempDirectory))
				{
					Spludlow.Archive.Extract(sourceZipFilename, tempDir.Path);
					foreach (string filename in Directory.GetFiles(tempDir.Path, "*", SearchOption.AllDirectories))
						File.SetAttributes(filename, FileAttributes.Normal);

					foreach (DataRow row in table.Select("ArchiveName = '" + archive + "'"))
					{
						string sha1 = (string)row["SHA1"];

						if (hashes.Contains(sha1) == true)
							continue;

						string romName = (string)row["RomName"];

						string sourceFilename = tempDir + @"\" + romName;

						if (File.Exists(sourceFilename) == false)
							throw new ApplicationException("sourceFilename: " + sourceFilename);

						string targetFilename = HashStoreFilename(sha1, storeDirectory, true);

						if (File.Exists(targetFilename) == true)
							throw new ApplicationException("targetFilename already there: " + targetFilename);

						File.Copy(sourceFilename, targetFilename);

						hashes.Add(sha1);
					}
				}
			}

			int addCount = hashes.Count - startCount;

			Spludlow.Log.Report("LoadRoms: " + romIndexFilename + ", submit:" + table.Rows.Count + ", add:" + addCount + ", stored:" + hashes.Count);
		}

		/// <summary>
		/// STEP 3 (Optional)
		/// After importing from a rom set source you can use this to remove everything that was taken from the source
		/// Will leave what's left from ".bad.txt" (Step 1) so you can go back and fix any rom zips that failed and run the import again
		/// 
		/// NOTE: if .bad.txt is not found the passed directory will simpily be deleted
		/// 
		/// NOTE: .bad.txt conatins full paths (so don't move things around)
		/// 
		/// NOTE: Rename the fixed rom directory before running step 1 on it again or the previous index file will be wiped out
		/// </summary>
		public static void DeleteImportedRoms(string directory)
		{
			string badFilename = directory + ".bad.txt";

			if (File.Exists(badFilename) == false)
			{
				Directory.Delete(directory, true);
			}
			else
			{
				HashSet<string> badFilenames = new HashSet<string>(File.ReadAllLines(badFilename));

				foreach (string filename in Directory.GetFiles(directory))
				{
					if (badFilenames.Contains(filename) == false)
						File.Delete(filename);
				}
			}
		}

		/// <summary>
		/// Perform steps 1 to 3 on all sub directories
		/// So you can do a big batch and go to the pub
		/// 
		/// NOTE: The temp directory should be big engough to hold the largest extacted zip
		/// 
		/// NOTE: If a rom zip fails due to the RAM disk not being big enough it will end up in .bat.txt so you can run it again without a RAM disk
		/// </summary>
		public static void FullImportSubDirectoryRomSets(string rootDirectory, string storeDirectory, string tempDirectory)
		{
			foreach (string directory in Directory.GetDirectories(rootDirectory))
			{
				IndexRomDirectory(directory, tempDirectory);

				string indexFilename = directory + ".txt";

				LoadRoms(indexFilename, storeDirectory, tempDirectory);

				DeleteImportedRoms(directory);
			}
		}


		public static void CreateMachineRomSets(string targetDirectory, bool reportMode)
		{
			string connectionString = "@MameMachines";
			string storeDirectory = Spludlow.Config.Get("Spludlow.Mame.HashStore.MachineRom");
			string tempDirectory = "";

			CreateMachineRomSets(connectionString, targetDirectory, storeDirectory, tempDirectory, reportMode);
		}

		public static void CreateMachineRomSets(string targetDirectory, bool reportMode, string tempDirectory)
		{
			string connectionString = "@MameMachines";
			string storeDirectory = Spludlow.Config.Get("Spludlow.Mame.HashStore.MachineRom");

			CreateMachineRomSets(connectionString, targetDirectory, storeDirectory, tempDirectory, reportMode);
		}

		/// <summary>
		/// Make all MAME machine rom set zips posible from the SAH1 store roms
		/// The target zip will be created even if roms are missing (see report)
		/// A text table report is saved in the target directory
		/// Use null for the temp dir or a RAM disk for speed
		/// </summary>
		public static void CreateMachineRomSets(string connectionString, string targetDirectory, string storeDirectory, string tempDirectory, bool reportMode)
		{
			Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create(connectionString);

			HashSet<string> storeHashes = ReadExistingHashes(storeDirectory);

			DataTable machineTable = database.Select("SELECT * FROM machine");
			DataTable romTable = database.Select("SELECT * FROM rom");

			DataTable resultsTable = Spludlow.Data.TextTable.ReadText(new string[]
			{
				"Machine	Description	Year	Parent	ParentCount	RomRowCount	ParentRomCount	ZipRomCount	MissingRomCount",
				"String		String		String	String	Int32		Int32		Int32			Int32		Int32",
			});

			DataTable missingRomTable = Spludlow.Data.TextTable.ReadText(new string[]
			{
				"Machine	Parent	SHA1",
				"String		String	String",
			});

			foreach (DataRow machineRow in machineTable.Rows)
			{
				int machine_id = (int)machineRow["machine_id"];
				DataRow[] romRows = romTable.Select("machine_id = " + machine_id);

				if (romRows.Length == 0)
					continue;

				string machineName = (string)machineRow["name"];

				string description = "";
				if (machineRow.IsNull("description") == false)
					description = (string)machineRow["description"];
				string year = "";
				if (machineRow.IsNull("year") == false)
					year = (string)machineRow["year"];

				string parent = "";
				if (machineRow.IsNull("romof") == false)
					parent = (string)machineRow["romof"];

				HashSet<string> parentHashes = new HashSet<string>();
				int parentCount = CheckParents(0, machineRow, parentHashes, romTable);
				
				int parentRomCount = 0;

				int missingRomCount = 0;

				List<DataRow> useRomRows = new List<DataRow>();

				foreach (DataRow romRow in romRows)
				{
					if (romRow.IsNull("sha1") == true)
						continue;
					string sha1 = ((string)romRow["sha1"]).ToUpper();

					if (storeHashes.Contains(sha1) == false)
					{
						missingRomTable.Rows.Add(machineName, parent, sha1);
						++missingRomCount;
						continue;
					}
					
					if (parentHashes.Contains(sha1) == true)
						++parentRomCount;
					else
						useRomRows.Add(romRow);
				}

				if (reportMode == false && useRomRows.Count > 0)
				{
					string targetZipFilename = targetDirectory + @"\" + machineName + ".zip";

					if (File.Exists(targetZipFilename) == false)
					{
						using (Spludlow.TempDirectory tempDir = new TempDirectory(tempDirectory))
						{
							foreach (DataRow romRow in useRomRows)
							{
								string sha1 = ((string)romRow["sha1"]).ToUpper();
								string name = (string)romRow["name"];
								long size = Int64.Parse((string)romRow["size"]);

								string sourceFilename = HashStoreFilename(sha1, storeDirectory, false);

								long fileSize = Spludlow.Io.Files.FileLength(sourceFilename);

								if (size != fileSize)
									throw new ApplicationException("Bad length: " + sha1 + ", actual:" + fileSize + ", record:" + size);

								string targetFilename = tempDir.Path + @"\" + name;

								if (File.Exists(targetFilename) == true)
								{
									//	need to check sha1 same !!! not sure why this is haperning?
									File.Delete(targetFilename);
									Spludlow.Log.Warning("Rom name already exists: " + machineName + ", " + name);
								}

								File.Copy(sourceFilename, targetFilename);
							}
							
							Spludlow.Archive.Create(targetZipFilename, tempDir.Path + @"\*");
						}
					}
				}

				resultsTable.Rows.Add(machineName, description, year, parent, parentCount, romRows.Length, parentRomCount, useRomRows.Count, missingRomCount);

			}

			string reportName = "_" + Spludlow.Text.TimeStamp();

			Spludlow.Data.TextTable.Write(targetDirectory + @"\" + reportName + ".txt", resultsTable);
			Spludlow.Data.TextTable.Write(targetDirectory + @"\" + reportName + ".NoRoms.txt", missingRomTable);

			int missingCount = 0;
			int missingTotal = 0;

			int numParents = 0;
			int missingCountParents = 0;
			int missingTotalParents = 0;

			foreach (DataRow row in resultsTable.Rows)
			{
				int missing = (int)row["MissingRomCount"];

				if (missing > 0)
					++missingCount;
				missingTotal += missing;

				int parentCount = (int)row["ParentCount"];

				if (parentCount == 0)
				{
					if (missing > 0)
						++missingCountParents;
					missingTotalParents += missing;

					++numParents;
				}
			}

			DataView viewParents = new DataView(resultsTable);
			viewParents.RowFilter = "ParentCount = 0 AND MissingRomCount > 0";
			viewParents.Sort = "Machine";

			DataView viewChildren = new DataView(resultsTable);
			viewChildren.RowFilter = "ParentCount > 0 AND MissingRomCount > 0";
			viewChildren.Sort = "Parent, Machine";

			StringBuilder text = new StringBuilder();
			text.AppendLine("Total Machines:				" + resultsTable.Rows.Count);
			text.AppendLine("All Machines missing ROMS:		" + missingCount);
			text.AppendLine("All machines complete ROMS:	" + (resultsTable.Rows.Count - missingCount));
			text.AppendLine("All missing ROMS:				" + missingTotal);
			text.AppendLine();
			text.AppendLine("Total Machines:				" + numParents);
			text.AppendLine("Parent Machines missing ROMS:	" + missingCountParents);
			text.AppendLine("Parent machines complete ROMS:	" + (numParents - missingCountParents));
			text.AppendLine("Parent missing ROMS:			" + missingTotalParents);

			Spludlow.Log.Report("CreateMachineRomSets", text.ToString(), viewParents, viewChildren, missingRomTable);

		}

		/// <summary>
		/// Recursively traverse ancestors to add sha1s to the HashSet
		/// Used to see what roms don't need including in a machine's set as parent(s) already have them
		/// </summary>
		private static int CheckParents(int count, DataRow machineRow, HashSet<string> parentHashes, DataTable romTable)
		{
			if (machineRow.IsNull("romof") == true)
				return count;

			string name = (string)machineRow["name"];
			string parent = (string)machineRow["romof"];

			DataTable table = machineRow.Table;
			DataRow[] rows = table.Select("name = '" + parent + "'");

			if (rows.Length != 1)
				throw new ApplicationException("CheckParentrs: " + name + ", parent:" + parent);

			DataRow[] romRows = romTable.Select("machine_id = " + rows[0]["machine_id"]);
			foreach (DataRow romRow in romRows)
			{
				if (romRow.IsNull("sha1") == true)
					continue;
				string sha1 = ((string)romRow["sha1"]).ToUpper();
				if (parentHashes.Contains(sha1) == false)
					parentHashes.Add(sha1);
			}

			return CheckParents(++count, rows[0], parentHashes, romTable);
		}


		/// <summary>
		/// Get a hash set of existing roms in the store
		/// </summary>
		public static HashSet<string> ReadExistingHashes(string storeDirectory)
		{
			HashSet<string> hashes = new HashSet<string>();

			foreach (string filename in Directory.GetFiles(storeDirectory, "*", SearchOption.AllDirectories))
				hashes.Add(Path.GetFileName(filename));

			return hashes;
		}


		/// <summary>
		/// Get SHA1 Store Filename
		/// Using first 2 chars for directory balancing (prevent too many files in single directory)
		/// </summary>
		public static string HashStoreFilename(string sha1, string storeDirectory, bool writeMode)
		{
			if (sha1.Length != 40)
				throw new ApplicationException("Bad sha1: " + sha1);

			StringBuilder path = new StringBuilder();
			path.Append(storeDirectory);

			path.Append(@"\");
			path.Append(sha1.Substring(0, 2));

			path.Append(@"\");
			path.Append(sha1);

			string filename = path.ToString();

			if (writeMode == true)
			{
				string directory = Path.GetDirectoryName(filename);
				if (Directory.Exists(directory) == false)
					Directory.CreateDirectory(directory);
			}

			return filename;
		}


		/// <summary>
		/// Example command text. Always do "SELECT * FROM machine" you can change the WHERE and ORDER BY to suit your needs
		/// Go for all machines in the 1980s that have no parents (the master that all derivatives reference, otherwise you get loads of variations of the same game)
		/// Order by manufacturer then year
		/// </summary>
		public static void MameBatFile1980sParents(string targetFilename, string connectionString)
		{
			string commandText = "SELECT * FROM machine WHERE ((machine.year LIKE '198%') AND (machine.romof IS NULL)) ORDER BY machine.manufacturer, machine.year, machine.name";

			MameBatFile(targetFilename, connectionString, commandText);
		}


		/// <summary>
		/// Create a BAT file to sequentially run a bunch of MAME games
		/// When in MAME press ESCAPE to quit to next game.
		/// If ESCAPE fails do ALT + TAB then right click on MAME in Windows TaskBar to close
		/// To stop do ALT + TAB then make sure the command prompt has focus and do CRTL + C to terminate the BAT
		/// Edit the BAT in notepad to chop out what you dont want
		/// </summary>
		public static void MameBatFile(string targetFilename, string connectionString, string commandText)
		{
			Spludlow.Data.IDAL database = Spludlow.Data.DAL.Create(connectionString);

			DataTable table = database.Select(commandText);

			StringBuilder result = new StringBuilder();

			foreach (DataRow row in table.Rows)
			{
				string name = (string)row["name"];
				string manufacturer = (string)row["manufacturer"];
				string year = (string)row["year"];
				string description = (string)row["description"];

				result.AppendLine("REM manufacturer:" + manufacturer + " year:" + year + " description:" + description + " name:" + name);
				result.AppendLine("mame64.exe " + name);
			}

			File.WriteAllText(targetFilename, result.ToString());
		}


		/// <summary>
		/// Merge hash store, add to master what haven't got from source
		/// </summary>
		public static void MergeHashStore(string masterDirectory, string sourceDirectory)
		{
			HashSet<string> masterHashes = ReadExistingHashes(masterDirectory);
			HashSet<string> sourceHashes = ReadExistingHashes(sourceDirectory);

			int count = 0;

			foreach (string sha1 in sourceHashes)
			{
				if (masterHashes.Contains(sha1) == true)
					continue;

				File.Copy(HashStoreFilename(sha1, sourceDirectory, false), HashStoreFilename(sha1, masterDirectory, true));

				++count;
			}

			Spludlow.Log.Report("MergeHashStore, master:" + masterHashes.Count + ", source:" + sourceHashes.Count + ", copy:" + count);
		}


		public static void ReportDiff(HashSet<string> listA, HashSet<string> listB, string nameA, string nameB, bool reportData)
		{
			HashSet<string> notInA = new HashSet<string>();
			HashSet<string> notInB = new HashSet<string>();
			HashSet<string> inBoth = new HashSet<string>();
			HashSet<string>[] lists = new HashSet<string>[] { notInA, notInB, inBoth };
			string[] lables = new string[] { "In @A, missing from @B", "In @B, missing from @A", "In both" };
		
			for (int index = 0; index < lables.Length; ++index)
			{
				lables[index] = lables[index].Replace("@A", nameA).Replace("@B", nameB);
			}

			foreach (string a in listA)
			{
				if (listB.Contains(a) == true)
					inBoth.Add(a);
				else
					notInB.Add(a);
			}
			foreach (string b in listB)
			{
				if (listA.Contains(b) == false)
					notInA.Add(b);
			}

			DataSet dataSet = new DataSet();

			if (reportData == true)
			{
				for (int index = 0; index < 3; ++index)
				{
					DataTable table = new DataTable();
					table.Columns.Add("Value", typeof(string));
					table.TableName = lables[index];
					foreach (string item in lists[index])
						table.Rows.Add(item);

					dataSet.Tables.Add(table);
				}
			}


			StringBuilder info = new StringBuilder();

			info.AppendLine(nameA + " Total " + listA.Count);
			info.AppendLine(nameB + " Total " + listB.Count);

			for (int index = 0; index < 3; ++index)
				info.AppendLine(lables[index] + " " + lists[index].Count);


			Spludlow.Log.Report("ReportDiff", info.ToString(), dataSet);
		}
	}
}