// Spludlow Software // Copyright © Samuel P. Ludlow 2020 All Rights Reserved // Distributed under the terms of the GNU General Public License version 3 // Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE // https://www.spludlow.co.uk/LICENCE.TXT // The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission // v1.14 using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Xml.Serialization; using System.Data; using System.Security.Cryptography; using System.Runtime.Serialization; using System.Runtime.Serialization.Formatters.Binary; namespace Spludlow { /// /// Simple hashing static methods. /// Use Hash: MD5, SHA1, or SHA256 /// Create from: byte[], string, Stream, or filename /// Results in: byte[], Base64, Hex, Hex with dashes /// For fast integraty checks I would recomend MD5, Base64=char(24), Hex=char(32) /// For a fingerprints I would recomend SHA1, Base64=char(28), Hex=char(40) /// Base64 is case sensitive so not ideal for database queries, having in urls, or filnames. (You may get case insensative collisions) /// Hex is generally the one you want to use in your applications /// Hex with Dash is better for human readability /// public class Hashing { // Thread handling notes: the .net hash objects are not thread safe, you have 3 choices: // 1. Create a hash object with each call inside using{}. Performance: Lots of object created and destroyed // 2. Create static hash object and lock{} with each use. Performance: hashing is effectivly queued onto a single thread // 3. Create hash object for each sperate thread with "[ThreadStatic]" (tested & works). [ThreadStatic] private static MD5 _MD5 = null; [ThreadStatic] private static SHA1Managed _SHA1Managed = null; [ThreadStatic] private static SHA256Managed _SHA256Managed = null; [ThreadStatic] private static CRC32 _CRC32 = null; //private static SHA384Managed _SHA384Managed; //private static SHA512Managed _SHA512Managed; private static Encoding _Encoding = Encoding.UTF8; public static void LogHashingReport() { DataTable table = Spludlow.Data.TextTable.ReadText(new string[] { "Name Speed LengthBytes LengthBase64 LengthHex LengthHexDash", "String Int32 Int32 Int32 Int32 Int32", "MD5 4 16 24 32 47", "SHA1 11 20 28 40 59", "SHA256 24 32 44 64 95", "SHA384 29 48 64 96 143", "SHA512 29 64 88 128 191", // Add CRC32 }); Spludlow.Log.Report("Hashing Comparison Table", table); } public static byte[] Compute(string hashType, byte[] data) { switch (hashType) { case "MD5": return MD5(data); case "SHA1": return SHA1(data); case "SHA256": return SHA256(data); case "CRC32": return CRC32(data); default: throw new ApplicationException("Hashing Unkown Hash type: " + hashType); } } public static byte[] Compute(string hashType, Stream stream) { switch (hashType) { case "MD5": return MD5(stream); case "SHA1": return SHA1(stream); case "SHA256": return SHA256(stream); case "CRC32": return CRC32(stream); default: throw new ApplicationException("Hashing Unkown Hash type: " + hashType); } } public static byte[] Compute(string hashType, string text) { switch (hashType) { case "MD5": return MD5(text); case "SHA1": return SHA1(text); case "SHA256": return SHA256(text); case "CRC32": return CRC32(text); default: throw new ApplicationException("Hashing Unkown Hash type: " + hashType); } } public static byte[] ComputeFile(string hashType, string filename) { switch (hashType) { case "MD5": return MD5File(filename); case "SHA1": return SHA1File(filename); case "SHA256": return SHA256File(filename); case "CRC32": return CRC32File(filename); default: throw new ApplicationException("Hashing Unkown Hash type: " + hashType); } } private static MemoryStream MakeSerializeStream(object instance) { MemoryStream memoryStream = new MemoryStream(); BinaryFormatter binaryFormatter = new BinaryFormatter(); binaryFormatter.Serialize(memoryStream, instance); memoryStream.Position = 0; return memoryStream; } public static string Base64(byte[] data) { return Convert.ToBase64String(data); } public static string Hex(byte[] data) { StringBuilder hex = new StringBuilder(); foreach (byte b in data) hex.Append(b.ToString("X2")); return hex.ToString(); } public static string HexDash(byte[] data) { return BitConverter.ToString(data); } // MD5 private static void MD5Make() { if (_MD5 == null) _MD5 = System.Security.Cryptography.MD5.Create(); } public static byte[] MD5(byte[] data) { MD5Make(); return _MD5.ComputeHash(data); } public static string MD5Base64(byte[] data) { MD5Make(); byte[] hash = _MD5.ComputeHash(data); return Convert.ToBase64String(hash); } public static string MD5Hex(byte[] data) { MD5Make(); byte[] hash = _MD5.ComputeHash(data); return Hex(hash); } public static string MD5HexDash(byte[] data) { MD5Make(); byte[] hash = _MD5.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] MD5(Stream stream) { MD5Make(); return _MD5.ComputeHash(stream); } public static string MD5Base64(Stream stream) { MD5Make(); byte[] hash = _MD5.ComputeHash(stream); return Convert.ToBase64String(hash); } public static string MD5Hex(Stream stream) { MD5Make(); byte[] hash = _MD5.ComputeHash(stream); return Hex(hash); } public static string MD5HexDash(Stream stream) { MD5Make(); byte[] hash = _MD5.ComputeHash(stream); return BitConverter.ToString(hash); } public static byte[] MD5(string text) { MD5Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); return _MD5.ComputeHash(data); } public static string MD5Base64(string text) { MD5Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _MD5.ComputeHash(data); return Convert.ToBase64String(hash); } public static string MD5Hex(string text) { MD5Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _MD5.ComputeHash(data); return Hex(hash); } public static string MD5HexDash(string text) { MD5Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _MD5.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] MD5File(string filename) { MD5Make(); using (FileStream stream = File.OpenRead(filename)) return _MD5.ComputeHash(stream); } public static string MD5Base64File(string filename) { MD5Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _MD5.ComputeHash(stream); return Convert.ToBase64String(hash); } } public static string MD5HexFile(string filename) { MD5Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _MD5.ComputeHash(stream); return Hex(hash); } } public static string MD5HexDashFile(string filename) { MD5Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _MD5.ComputeHash(stream); return BitConverter.ToString(hash); } } public static byte[] MD5Object(object instance) // Add object for others { MD5Make(); using (Stream stream = MakeSerializeStream(instance)) return _MD5.ComputeHash(stream); } public static string MD5Base64Object(object instance) { MD5Make(); using (Stream stream = MakeSerializeStream(instance)) { byte[] hash = _MD5.ComputeHash(stream); return Convert.ToBase64String(hash); } } public static string MD5HexObject(object instance) { MD5Make(); using (Stream stream = MakeSerializeStream(instance)) { byte[] hash = _MD5.ComputeHash(stream); return Hex(hash); } } public static string MD5HexDashObject(object instance) { MD5Make(); using (Stream stream = MakeSerializeStream(instance)) { byte[] hash = _MD5.ComputeHash(stream); return BitConverter.ToString(hash); } } // SHA1 private static void SHA1Make() { if (_SHA1Managed == null) _SHA1Managed = new SHA1Managed(); } public static byte[] SHA1(byte[] data) { SHA1Make(); return _SHA1Managed.ComputeHash(data); } public static string SHA1Base64(byte[] data) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(data); return Convert.ToBase64String(hash); } public static string SHA1Hex(byte[] data) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(data); return Hex(hash); } public static string SHA1HexDash(byte[] data) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] SHA1(Stream stream) { SHA1Make(); return _SHA1Managed.ComputeHash(stream); } public static string SHA1Base64(Stream stream) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(stream); return Convert.ToBase64String(hash); } public static string SHA1Hex(Stream stream) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(stream); return Hex(hash); } public static string SHA1HexDash(Stream stream) { SHA1Make(); byte[] hash = _SHA1Managed.ComputeHash(stream); return BitConverter.ToString(hash); } public static byte[] SHA1(string text) { SHA1Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); return _SHA1Managed.ComputeHash(data); } public static string SHA1Base64(string text) { SHA1Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA1Managed.ComputeHash(data); return Convert.ToBase64String(hash); } public static string SHA1Hex(string text) { SHA1Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA1Managed.ComputeHash(data); return Hex(hash); } public static string SHA1HexDash(string text) { SHA1Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA1Managed.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] SHA1File(string filename) { SHA1Make(); using (FileStream stream = File.OpenRead(filename)) return _SHA1Managed.ComputeHash(stream); } public static string SHA1Base64File(string filename) { SHA1Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA1Managed.ComputeHash(stream); return Convert.ToBase64String(hash); } } public static string SHA1HexFile(string filename) { SHA1Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA1Managed.ComputeHash(stream); return Hex(hash); } } public static string SHA1HexDashFile(string filename) { SHA1Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA1Managed.ComputeHash(stream); return BitConverter.ToString(hash); } } // SHA256 private static void SHA256Make() { if (_SHA256Managed == null) _SHA256Managed = new SHA256Managed(); } public static byte[] SHA256(byte[] data) { SHA256Make(); return _SHA256Managed.ComputeHash(data); } public static string SHA256Base64(byte[] data) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(data); return Convert.ToBase64String(hash); } public static string SHA256Hex(byte[] data) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(data); return Hex(hash); } public static string SHA256HexDash(byte[] data) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] SHA256(Stream stream) { SHA256Make(); return _SHA256Managed.ComputeHash(stream); } public static string SHA256Base64(Stream stream) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(stream); return Convert.ToBase64String(hash); } public static string SHA256Hex(Stream stream) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(stream); return Hex(hash); } public static string SHA256HexDash(Stream stream) { SHA256Make(); byte[] hash = _SHA256Managed.ComputeHash(stream); return BitConverter.ToString(hash); } public static byte[] SHA256(string text) { SHA256Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); return _SHA256Managed.ComputeHash(data); } public static string SHA256Base64(string text) { SHA256Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA256Managed.ComputeHash(data); return Convert.ToBase64String(hash); } public static string SHA256Hex(string text) { SHA256Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA256Managed.ComputeHash(data); return Hex(hash); } public static string SHA256HexDash(string text) { SHA256Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _SHA256Managed.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] SHA256File(string filename) { SHA256Make(); using (FileStream stream = File.OpenRead(filename)) return _SHA256Managed.ComputeHash(stream); } public static string SHA256Base64File(string filename) { SHA256Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA256Managed.ComputeHash(stream); return Convert.ToBase64String(hash); } } public static string SHA256HexFile(string filename) { SHA256Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA256Managed.ComputeHash(stream); return Hex(hash); } } public static string SHA256HexDashFile(string filename) { SHA256Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _SHA256Managed.ComputeHash(stream); return BitConverter.ToString(hash); } } // CRC32 private static void CRC32Make() { if (_CRC32 == null) _CRC32 = new CRC32(); } public static byte[] CRC32(byte[] data) { CRC32Make(); return _CRC32.ComputeHash(data); } public static string CRC32Base64(byte[] data) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(data); return Convert.ToBase64String(hash); } public static string CRC32Hex(byte[] data) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(data); return Hex(hash); } public static string CRC32HexDash(byte[] data) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] CRC32(Stream stream) { CRC32Make(); return _CRC32.ComputeHash(stream); } public static string CRC32Base64(Stream stream) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(stream); return Convert.ToBase64String(hash); } public static string CRC32Hex(Stream stream) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(stream); return Hex(hash); } public static string CRC32HexDash(Stream stream) { CRC32Make(); byte[] hash = _CRC32.ComputeHash(stream); return BitConverter.ToString(hash); } public static byte[] CRC32(string text) { CRC32Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); return _CRC32.ComputeHash(data); } public static string CRC32Base64(string text) { CRC32Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _CRC32.ComputeHash(data); return Convert.ToBase64String(hash); } public static string CRC32Hex(string text) { CRC32Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _CRC32.ComputeHash(data); return Hex(hash); } public static string CRC32HexDash(string text) { CRC32Make(); byte[] data = Spludlow.Text.GetBytes(text, _Encoding); byte[] hash = _CRC32.ComputeHash(data); return BitConverter.ToString(hash); } public static byte[] CRC32File(string filename) { CRC32Make(); using (FileStream stream = File.OpenRead(filename)) return _CRC32.ComputeHash(stream); } public static string CRC32Base64File(string filename) { CRC32Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _CRC32.ComputeHash(stream); return Convert.ToBase64String(hash); } } public static string CRC32HexFile(string filename) { CRC32Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _CRC32.ComputeHash(stream); return Hex(hash); } } public static string CRC32HexDashFile(string filename) { CRC32Make(); using (FileStream stream = File.OpenRead(filename)) { byte[] hash = _CRC32.ComputeHash(stream); return BitConverter.ToString(hash); } } /// /// Prepare test files /// Spludlow.Io.TestFiles.MakeHashedSHA1Set(@"E:\TEST", 128, 4, 32); /// Run test, will log report /// public static void ThreadHashTest(string directory, int threads, int passes) { DataTable results = Spludlow.Data.TextTable.ReadText(new string[] { "Pass Thread FileHash Status", "Int32 Int32 String String", }); string[] filenames = Directory.GetFiles(directory, "*.dat"); Dictionary dataSet = new Dictionary(); foreach (string filename in filenames) { string hash = Path.GetFileNameWithoutExtension(filename); dataSet.Add(hash, File.ReadAllBytes(filename)); } Random random = new Random(); for (int pass = 0; pass < passes; ++pass) { List items = new List(dataSet.Keys); while (items.Count > 0) { Task[] tasks = new Task[threads]; for (int index = 0; index < threads; ++index) { int threadId = index; tasks[index] = new Task(() => ThreadHashTestWorker(items, dataSet, random, results, pass, threadId)); } foreach (Task task in tasks) task.Start(); Task.WaitAll(tasks); } } Spludlow.Log.Finish("ThreadHashTest: " + results.Rows.Count, results); } private static void ThreadHashTestWorker(List items, Dictionary dataSet, Random random, DataTable results, int pass, int threadId) { string fileHash; lock (items) { if (items.Count == 0) return; int index = random.Next(items.Count); fileHash = items[index]; items.RemoveAt(index); } byte[] data = dataSet[fileHash]; string calcHash = Spludlow.Hashing.SHA1Hex(data); string status = ""; if (calcHash != fileHash) status = "FAIL:" + calcHash; results.Rows.Add(pass, threadId, fileHash, status); } } }