// Spludlow Software // Copyright © Samuel P. Ludlow 2020 All Rights Reserved // Distributed under the terms of the GNU General Public License version 3 // Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE // https://www.spludlow.co.uk/LICENCE.TXT // The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission // v1.14 using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Data; // fix bad sources !!!! wiping out good history namespace Spludlow { public class Backup { private string TempPath = null; private Dictionary SourceItems = new Dictionary(); private Dictionary TargetItems = new Dictionary(); private Dictionary> SourceGroups = new Dictionary>(); private Dictionary> TargetGroups = new Dictionary>(); private Dictionary> ActionItems = new Dictionary>(); private Dictionary FtpClients = new Dictionary(); // Result "" = ok, "Waring", "Target Error" DataTable ReportTable = Spludlow.Data.TextTable.ReadText(new string[] { "Time Result Subject Body", "DateTime String String String", }); private void Report(string result, string subject, string body) { this.ReportTable.Rows.Add(new object[] { DateTime.Now, result, subject, body }); } private bool TestMode = false; public Backup() { } public void Run(string profileName) { this.Run(profileName, false); } public void Run(string profileName, bool dummyRun) { string configFilename = Spludlow.Config.ProgramData + @"\Config\Backup.txt"; this.Run(profileName, configFilename, dummyRun); } public void Run(string profileName, string configFilename, bool testMode) { profileName = profileName.ToUpper(); this.TestMode = testMode; try { this.TempPath = Spludlow.Config.Get("Spludlow.BackupTemp", true); if (this.TempPath != null && Directory.Exists(this.TempPath) == false) throw new ApplicationException("Backup; Temp Directory specified in config key does not exist: " + this.TempPath); if (this.TestMode == true) this.Report("", "TEST MODE!", ""); this.Report("", "Starting: " + profileName, ""); this.ReadConfigFile(configFilename); this.Report("", "Config Read, Starting Main", ""); this.ProcessActions(profileName); this.Report("", "Finished: " + profileName, ""); string logSubject = "Backup; Complete. profile:" + profileName + ", configFilename: " + configFilename + ", TestMode:" + testMode; if (this.ReportTable.Select("Result LIKE '%error%'").Length > 0) { Spludlow.Log.Error(logSubject, new object[] { this.ReportTable }); } else { if (this.ReportTable.Select("Result LIKE '%warning%'").Length > 0) Spludlow.Log.Warning(logSubject, new object[] { this.ReportTable }); else Spludlow.Log.Finish(logSubject, new object[] { this.ReportTable }); } } catch (Exception ee) { this.Report("Fatal Error", ee.Message, ee.ToString()); Spludlow.Log.Error("Backup; Fatal Error, " + ee.Message + ", profile:" + profileName + ", configFilename: " + configFilename, new object[] { ee, this.ReportTable }); } } private void ReadConfigFile(string filename) { Dictionary> items = new Dictionary>(); // [SOURCES] [TARGETS] All others are actions string currentSection = null; foreach (string rawLine in File.ReadAllLines(filename)) { string line = rawLine.Trim(); if (line.Length == 0 || line.StartsWith("#") == true) continue; string[] words = Spludlow.Text.Split(line, '\t', true, false); words[0] = words[0].ToUpper(); if (words[0].StartsWith("[") == true) { currentSection = words[0].Trim(new char[] { '[', ']' }); continue; } if (currentSection == null) throw new ApplicationException("Backup, ReadConfigFile; Lines not inside section"); if (items.ContainsKey(currentSection) == false) items.Add(currentSection, new List()); items[currentSection].Add(words); } if (items.ContainsKey("SOURCES") == false) throw new ApplicationException("Backup, ReadConfigFile; no [SOURCES] section"); if (items.ContainsKey("TARGETS") == false) throw new ApplicationException("Backup, ReadConfigFile; no [TARGETS] section"); if (items.Keys.Count == 2) Spludlow.Log.Warning("Backup, ReadConfigFile; No action sections defined, nothing will be backed up"); foreach (string section in items.Keys) { switch (section) { case "SOURCES": this.ReadConfigSources(items[section]); break; case "TARGETS": this.ReadConfigTargets(items[section]); break; default: this.ReadConfigActions(section, items[section]); break; } } } private void ReadConfigSources(List lines) { // perform checks for dupliate keys, otherwise will throw when aading to dictionary !"!!!!! foreach (string[] words in lines) { //words[1] = words[1].ToUpper(); switch (words[0]) { case "FILE": if (words.Length != 3) throw new ApplicationException("Backup, ReadConfigFile, Sources FILE line must have 3 words 'FILE ' has:" + words.Length); this.SourceItems.Add(words[1], words); break; case "DATA": switch (words.Length) { case 4: this.SourceItems.Add(words[1], new string[] { words[0], words[1], words[2], words[3], null }); break; case 5: this.SourceItems.Add(words[1], words); break; default: throw new ApplicationException("Backup, ReadConfigFile, Sources DATA line must have 4 or 5 words 'DATA ' has:" + words.Length); } break; case "GROUP": if (words.Length < 3) throw new ApplicationException("Backup, ReadConfigFile, Sources GROUP line must have at least 3 words 'GROUP ' has:" + words.Length); string key = words[1]; this.SourceGroups.Add(key, new List()); for (int index = 2; index < words.Length; ++index) this.SourceGroups[key].Add(words[index]); // .ToUpper() break; default: throw new ApplicationException("Backup, ReadConfigFile, Sources Section; Unknown line type: " + words[0]); } } foreach (string key in this.SourceGroups.Keys) if (this.SourceItems.ContainsKey(key) == true) throw new ApplicationException("Backup, ReadConfigFile, Sources Section; Group key with same name as source item key: " + key); } private void ReadConfigTargets(List lines) { // perform checks for dupliate keys, otherwise will throw when adding to dictionary !"!!!!! foreach (string[] words in lines) { //words[1] = words[1].ToUpper(); switch (words[0]) { case "DIR": if (words.Length != 3) throw new ApplicationException("Backup, ReadConfigTargets, Targets DIR line must have 3 words 'DIR ' has:" + words.Length); this.TargetItems.Add(words[1], words); break; case "FTP": if (words.Length != 4) throw new ApplicationException("Backup, ReadConfigTargets, Targets FTP line must have 4 words 'FTP ' has:" + words.Length); this.TargetItems.Add(words[1], words); break; case "WEB": if (words.Length != 4) throw new ApplicationException("Backup, ReadConfigTargets, Targets WEB line must have 4 words 'WEB ' has:" + words.Length); this.TargetItems.Add(words[1], words); break; case "GROUP": if (words.Length < 3) throw new ApplicationException("Backup, ReadConfigTargets, Targets GROUP line must have at least 3 words 'GROUP ' has:" + words.Length); string key = words[1]; this.TargetGroups.Add(key, new List()); for (int index = 2; index < words.Length; ++index) this.TargetGroups[key].Add(words[index]); // .ToUpper() break; default: throw new ApplicationException("Backup, ReadConfigTargets, Targets Section; Unknown line type: " + words[0]); } } foreach (string key in this.TargetGroups.Keys) if (this.TargetItems.ContainsKey(key) == true) throw new ApplicationException("Backup, ReadConfigFile, Targets Section; Group key with same name as target item key: " + key); } private void ReadConfigActions(string section, List lines) { this.ActionItems.Add(section, new List()); foreach (string[] words in lines) { if (words[0] != "BACKUP" && words[0] != "ARCHIVE") throw new ApplicationException("Backup, ReadConfigActions, Actions; Unknown line type: " + words[0] + ", section:" + section); switch (words.Length) { case 5: this.ActionItems[section].Add(new string[] { words[0], words[1], words[2], words[3], words[4], null }); break; case 6: this.ActionItems[section].Add(words); break; default: throw new ApplicationException("Backup, ReadConfigActions, " + words[0] + " Action line must have 5 or 6 words '" + words[0] + " ' has:" + words.Length); } } } private void ProcessActions(string profileName) { for (int actionIndex = 0; actionIndex < this.ActionItems[profileName].Count; ++actionIndex) { try { string[] words = this.ActionItems[profileName][actionIndex]; string[] sources = this.ResolveGroups(words[1], this.SourceItems, this.SourceGroups); string[] targets = this.ResolveGroups(words[2], this.TargetItems, this.TargetGroups); string subPath = words[3].Replace('\\', '/'); subPath = "/" + subPath.Trim(new char[] { '/' }); int history; bool dontCompress = false; if (words[4].EndsWith("*") == true) { words[4] = words[4].Substring(0, words[4].Length - 1); dontCompress = true; } history = Int32.Parse(words[4]); string password = words[5]; switch (words[0]) { case "BACKUP": this.BackupAction(sources, targets, subPath, history, dontCompress, password); break; case "ARCHIVE": this.ArchiveAction(sources, targets, subPath, history, dontCompress, password); break; default: throw new ApplicationException("Backup, ProcessActions; Unknown action type: " + words[0]); } this.Report("", "Action Complete: profile:" + profileName + ", actionIndex:" + actionIndex, ""); } catch (Exception ee) { this.Report("Action Error", "Profile:" + profileName + ", actionIndex:" + actionIndex + ", " + ee.Message, ee.ToString()); } } } private string[] ResolveGroups(string key, Dictionary items, Dictionary> groups) { if (items.ContainsKey(key)) return new string[] { key }; if (groups.ContainsKey(key) == false) throw new ApplicationException("Backup, ResolveGroups; Key not in items or group: " + key); return groups[key].ToArray(); } private void BackupAction(string[] sourceKeys, string[] targetKeys, string subPath, int keepHistory, bool dontCompress, string password) { using (TempDirectory tempDir = new TempDirectory(this.TempPath)) { foreach (string sourceKey in sourceKeys) { try { foreach (string tempArchiveFilename in this.MakeSourceArchives(sourceKey, dontCompress, password, tempDir.Path, keepHistory)) { foreach (string targetKey in targetKeys) { string[] words = this.TargetItems[targetKey]; try { this.BackupTarget(words, targetKey, tempArchiveFilename, subPath, keepHistory); } catch (Exception ee) { this.Report("Target Error", "Target: " + targetKey + ", " + ee.Message, ee.ToString()); } } // Delete temp archive, save space !!!! here } } catch (Exception ee) { this.Report("Source Error", "Source: " + sourceKey + ", " + ee.Message, ee.ToString()); } } } } private void ArchiveAction(string[] sourceKeys, string[] targetKeys, string subPath, int keepHistory, bool dontCompress, string password) { foreach (string sourceKey in sourceKeys) { string[] words = this.SourceItems[sourceKey]; string sourceType = words[0]; string sourceDirectory = words[2]; if (sourceType != "FILE") { this.Report("Archive Error", "Sourcekey: " + sourceKey, "The ARCHIVE action can only be applied to FILE sources not: " + sourceType); continue; } string firstArchive = null; foreach (string targetKey in targetKeys) { try { words = this.TargetItems[targetKey]; string targetType = words[0]; if (firstArchive == null) { if (targetType != "DIR") { this.Report("Archive Error", "Sourcekey: " + sourceKey + ", TargetKey: " + targetKey, "The ARCHIVE actions first target must be a DIR type not: " + sourceType); continue; } string firstTargetDirectory = words[2]; if (subPath != "/") firstTargetDirectory += subPath; firstTargetDirectory = firstTargetDirectory.Replace("/", "\\"); this.PurgeDirBefore(firstTargetDirectory, keepHistory, this.MakeDirectoryArchiveFilename(sourceKey, firstTargetDirectory, keepHistory)); string[] firstFilenames = this.MakeSourceArchives(sourceKey, dontCompress, password, firstTargetDirectory, keepHistory); if (firstFilenames.Length != 1) throw new ApplicationException("Backup, Archive; First MakeSourceArchives did not return 1 filename: " + firstFilenames.Length); firstArchive = firstFilenames[0]; this.PurgeDirAfter(firstTargetDirectory, keepHistory, firstArchive); } else { this.BackupTarget(words, targetKey, firstArchive, subPath, keepHistory); } } catch (Exception ee) { this.Report("Archive Target Error", "Sourcekey: " + sourceKey + ", TargetKey: " + targetKey + ", " + ee.Message, ee.ToString()); } } } } private string BackupTarget(string[] words, string targetKey, string tempArchiveFilename, string subPath, int keepHistory) { string targetDir; string targetFilename; string testFilename; switch (words[0]) { case "DIR": targetDir = words[2]; if (subPath != "/") targetDir += subPath; targetDir = targetDir.Replace("/", "\\"); targetFilename = targetDir + @"\" + Path.GetFileName(tempArchiveFilename); if (this.TestMode == false) { this.PurgeDirBefore(targetDir, keepHistory, targetFilename); File.Copy(tempArchiveFilename, targetFilename); this.PurgeDirAfter(targetDir, keepHistory, targetFilename); } else { testFilename = targetFilename + ".dummy"; Directory.GetFiles(targetDir); File.WriteAllText(testFilename, testFilename); File.Delete(testFilename); } break; case "FTP": targetDir = words[2]; if (subPath != "/") targetDir += subPath; targetDir = targetDir.Replace("\\", "/"); targetFilename = targetDir + "/" + Path.GetFileName(tempArchiveFilename); if (this.FtpClients.ContainsKey(targetKey) == false) this.FtpClients.Add(targetKey, new Net.Ftp(words[3])); Spludlow.Net.Ftp ftp = this.FtpClients[targetKey]; if (this.TestMode == false) { this.PurgeFtpBefore(ftp, targetDir, keepHistory, targetFilename); ftp.UploadFile(tempArchiveFilename, targetFilename); this.PurgeFtpAfter(ftp, targetDir, keepHistory, targetFilename); } else { testFilename = targetFilename + ".dummy"; ftp.ListDirectory(targetDir); ftp.UploadText(testFilename, testFilename); ftp.DeleteFile(testFilename); } break; case "WEB": string host = words[2]; targetDir = words[3]; if (subPath != "/") targetDir += subPath; targetDir = targetDir.Replace("/", "\\"); targetFilename = targetDir + @"\" + Path.GetFileName(tempArchiveFilename); // same as dir!!!! if (this.TestMode == false) { this.PurgeWebBefore(host, targetDir, keepHistory, targetFilename); Spludlow.Call.Upload(host, tempArchiveFilename, targetFilename); this.PurgeWebAfter(host, targetDir, keepHistory, targetFilename); } else { testFilename = targetFilename + ".dummy"; Spludlow.RemoteIO.DirectoryGetFiles(host, targetDir); Spludlow.RemoteIO.FileWriteAllText(host, testFilename, testFilename); Spludlow.RemoteIO.FileDelete(host, testFilename); } break; default: throw new ApplicationException("Backup, BackupTarget; Unknow target type: " + words[0]); } this.Report("", "Backed up target: " + targetKey, tempArchiveFilename + " -> " + targetFilename); return targetFilename; } private string MakeDirectoryArchiveFilename(string sourceKey, string tempDir, int keepHistory) { string stamp = ""; if (keepHistory != 0) stamp = "_" + Spludlow.Text.TimeStamp(); return tempDir + @"\" + sourceKey + stamp + ".7z"; } private string[] MakeSourceArchives(string sourceKey, bool dontCompress, string password, string tempDir, int keepHistory) { List resultFilenames = new List(); string[] words = this.SourceItems[sourceKey]; string stamp; string warnings; string testFilename; switch (words[0]) { case "FILE": string sourcePath = words[2]; string resultFilename = this.MakeDirectoryArchiveFilename(sourceKey, tempDir, keepHistory); if (this.TestMode == false) { warnings = this.CreateArchive(resultFilename, sourcePath, password, dontCompress); if (warnings != null) this.Report("Archive Warning", "File System sourceKey: " + sourceKey, warnings); } else { testFilename = resultFilename + ".dummy"; if (Directory.Exists(sourcePath) == true) Directory.GetFiles(sourcePath); else File.GetAttributes(sourcePath); Directory.GetFiles(tempDir); File.WriteAllText(testFilename, testFilename); File.Delete(testFilename); } resultFilenames.Add(resultFilename); break; case "DATA": string databaseTempDir = words[2]; string connectionString = words[3]; string database = words[4]; Spludlow.Data.IDAL dal = Spludlow.Data.DAL.Create(connectionString); List databaseNameKeyNames = new List(); if (database == "*") // Temp dir must have enough space for all !!!!!!!!!!!!!!! { foreach (string databaseName in dal.DatabaseList()) databaseNameKeyNames.Add(new string[] { databaseName, sourceKey + "-" + databaseName }); } else { if (database == null) { if (dal.CurrentDatabase() == null) throw new ApplicationException("Backup, MakeSourceArchives; Database name not supplied or in connection string"); } else { dal.ChangeDatabase(database); } databaseNameKeyNames.Add(new string[] { dal.CurrentDatabase(), sourceKey }); } foreach (string[] databaseNameKeyName in databaseNameKeyNames) { string databaseName = databaseNameKeyName[0]; string keyName = databaseNameKeyName[1]; stamp = ""; if (keepHistory != 0) stamp = "_" + Spludlow.Text.TimeStamp(); // can get stray backup files here !!!! do all inside tempdir then wait and rtemove at end !!! string databaseTempFilename = databaseTempDir + @"\" + keyName + stamp + ".BAK"; // May want other extentions for example .SQL for MySQL !!!!!!!!! string tempFilename = tempDir + @"\" + Path.GetFileName(databaseTempFilename) + ".7z"; try { dal.ChangeDatabase(databaseName); if (this.TestMode == false) { dal.Backup(databaseTempFilename); int attempt = 0; while (File.Exists(databaseTempFilename) == false && ++attempt <= 10) { Spludlow.Log.Warning("Backup, MakeSourceArchives; Waiting for database backup file, attempt: " + attempt + ", file: " + databaseTempFilename); System.Threading.Thread.Sleep(3 * 1000); } if (File.Exists(databaseTempFilename) == false) throw new ApplicationException("Backup, MakeSourceArchives; database backup file never arived: " + databaseTempFilename); // Backup file can apear after command finihsed warnings = this.CreateArchive(tempFilename, databaseTempFilename, password, dontCompress); if (warnings != null) this.Report("Archive Warning", "Database sourceKey: " + sourceKey + ", database: " + databaseName, warnings); } else { testFilename = tempFilename + ".dummy"; Directory.GetFiles(databaseTempDir); Directory.GetFiles(tempDir); File.WriteAllText(testFilename, testFilename); File.Delete(testFilename); } resultFilenames.Add(tempFilename); } finally { if (File.Exists(databaseTempFilename) == true) File.Delete(databaseTempFilename); } } break; } foreach (string resultFilename in resultFilenames) this.Report("", "Created source archive, sourceKey: " + sourceKey, resultFilename); return resultFilenames.ToArray(); } private string CreateArchive(string tempFilename, string databaseTempFilename, string password, bool dontCompress) { if (password != null && password.StartsWith("@") == true) password = Spludlow.Credentials.GetCredential(password.Substring(1)).Password; return Spludlow.Archive.Create(tempFilename, databaseTempFilename, password, dontCompress); } private void PurgeDirBefore(string targetDir, int keepHistory, string targetFilename) { if (keepHistory == 0 && File.Exists(targetFilename) == true) File.Delete(targetFilename); if (keepHistory < 0) this.PurgeDir(targetDir, keepHistory, targetFilename); } private void PurgeDirAfter(string targetDir, int keepHistory, string targetFilename) { if (keepHistory > 0) this.PurgeDir(targetDir, keepHistory, targetFilename); } private int PurgeDir(string targetDir, int keepHistory, string targetFilename) { string[] filenames = this.PurgeFiles(Directory.GetFiles(targetDir), keepHistory, targetFilename); foreach (string filename in filenames) File.Delete(filename); return filenames.Length; } private void PurgeFtpBefore(Spludlow.Net.Ftp ftp, string targetDir, int keepHistory, string targetFilename) { if (keepHistory == 0 && ftp.FileExists(targetFilename) == true) ftp.DeleteFile(targetFilename); if (keepHistory < 0) this.PurgeFtp(ftp, targetDir, keepHistory, targetFilename); } private void PurgeFtpAfter(Spludlow.Net.Ftp ftp, string targetDir, int keepHistory, string targetFilename) { if (keepHistory > 0) this.PurgeFtp(ftp, targetDir, keepHistory, targetFilename); } private int PurgeFtp(Spludlow.Net.Ftp ftp, string targetDir, int keepHistory, string targetFilename) { List dirList = new List(); foreach (string filename in ftp.ListDirectory(targetDir)) dirList.Add(targetDir + "/" + filename); string[] filenames = this.PurgeFiles(dirList.ToArray(), keepHistory, targetFilename); foreach (string filename in filenames) ftp.DeleteFile(filename); return filenames.Length; } private void PurgeWebBefore(string host, string targetDir, int keepHistory, string targetFilename) { if (keepHistory == 0 && Spludlow.RemoteIO.FileExists(host, targetFilename) == true) Spludlow.RemoteIO.FileDelete(host, targetFilename); if (keepHistory < 0) this.PurgeWeb(host, targetDir, keepHistory, targetFilename); } private void PurgeWebAfter(string host, string targetDir, int keepHistory, string targetFilename) { if (keepHistory > 0) this.PurgeWeb(host, targetDir, keepHistory, targetFilename); } private int PurgeWeb(string host, string targetDir, int keepHistory, string targetFilename) { string[] filenames = this.PurgeFiles(Spludlow.RemoteIO.DirectoryGetFiles(host, targetDir), keepHistory, targetFilename); foreach (string filename in filenames) Spludlow.RemoteIO.FileDelete(host, filename); return filenames.Length; } private int StampLength = Spludlow.Text.TimeStamp().Length; private string[] PurgeFiles(string[] targetFilenames, int keepHistory, string targetFilename) { string name = Path.GetFileNameWithoutExtension(targetFilename); name = name.Substring(0, name.Length - this.StampLength); List targets = new List(); foreach (string targetPath in targetFilenames) { string filename = Path.GetFileName(targetPath); if (filename.StartsWith(name) == true && targetPath.Length == targetFilename.Length) targets.Add(targetPath); } targets.Sort(); if (keepHistory < 0) keepHistory = (keepHistory * -1) - 1; List removeList = new List(); while (targets.Count > keepHistory) { removeList.Add(targets[0]); targets.RemoveAt(0); } foreach (string removeFilename in removeList) this.Report("", "Purge File", removeFilename); return removeList.ToArray(); } } }