// Spludlow Software // Copyright © Samuel P. Ludlow 2020 All Rights Reserved // Distributed under the terms of the GNU General Public License version 3 // Distributed WITHOUT ANY WARRANTY; without implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE // https://www.spludlow.co.uk/LICENCE.TXT // The Spludlow logo is a registered trademark of Samuel P. Ludlow and may not be used without permission // v1.14 using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.IO; using System.Data; using System.Xml; using System.Xml.Linq; namespace Spludlow.Mame { public class MameDataImport { private static MameConfiguration _Config = new MameConfiguration(); public static string Import() { StringBuilder text = new StringBuilder(); string xmlCacheDirectory = _Config.DatabaseDirectory; string[] links = ExtractBinSrcTxtLinks(); string binArchiveUrl = links[0]; string release = Path.GetFileName(binArchiveUrl); release = Spludlow.Text.ChopBetween(release, "mame", "b", false); release = "0." + release.Substring(1); string binArchiveFilename = xmlCacheDirectory + @"\" + Path.GetFileName(binArchiveUrl); string machineXmlFilename = xmlCacheDirectory + @"\" + release + "-machine.xml"; string softwareXmlFilename = xmlCacheDirectory + @"\" + release + "-software.xml"; bool requireMachineUpdate = RequireDatabaseUpdate(MameDatabase.DatabaseNameMachine, release); bool requireSoftwareUpdate = RequireDatabaseUpdate(MameDatabase.DatabaseNameSoftware, release); if (requireMachineUpdate == false) text.AppendLine("Already have Machine database: " + release); if (requireSoftwareUpdate == false) text.AppendLine("Already have Software database: " + release); if (requireMachineUpdate == true || requireSoftwareUpdate == true) { if (File.Exists(binArchiveFilename) == false) Spludlow.Net.Http.GetDataFile(binArchiveUrl, binArchiveFilename); using (Spludlow.TempDirectory tempDir = new TempDirectory()) { string extractDirectory = tempDir.Path + @"\extract"; Directory.CreateDirectory(extractDirectory); Spludlow.Archive.Extract(binArchiveFilename, extractDirectory); string binFilename = extractDirectory + @"\mame.exe"; if (File.Exists(binFilename) == false) throw new ApplicationException("Mame Update Extract Binary List Data binFilename not there: " + binFilename); DateTime releaseDate = File.GetLastWriteTimeUtc(binFilename); Spludlow.SpawnProcess.ProcessExitInfo exitInfo; if (File.Exists(machineXmlFilename) == false) { exitInfo = Spludlow.SpawnProcess.Run(binFilename, "-listxml", extractDirectory, null, 0, false, null, Encoding.UTF8); if (exitInfo.ExitCode != 0) throw new ApplicationException("Error -listxml " + binFilename); File.WriteAllText(machineXmlFilename, exitInfo.StandardOutput, new UTF8Encoding(false)); File.SetLastWriteTimeUtc(machineXmlFilename, releaseDate); } if (requireMachineUpdate == true) { LoadDatabase(machineXmlFilename, MameDatabase.DatabaseNameMachine, "mame", release); text.AppendLine("Processed Machine database: " + release); } if (File.Exists(softwareXmlFilename) == false) { string listsoftwareFilename = tempDir.Path + @"\listsoftware.xml"; exitInfo = Spludlow.SpawnProcess.Run(binFilename, "-listsoftware", extractDirectory, null, 0, false, null, Encoding.UTF8); if (exitInfo.ExitCode != 0) throw new ApplicationException("Error -listsoftware " + binFilename); File.WriteAllText(listsoftwareFilename, exitInfo.StandardOutput, new UTF8Encoding(false)); CombineSoftwareList(extractDirectory + @"\hash", listsoftwareFilename, softwareXmlFilename); File.SetLastWriteTimeUtc(softwareXmlFilename, releaseDate); } if (requireSoftwareUpdate == true) { LoadDatabase(softwareXmlFilename, MameDatabase.DatabaseNameSoftware, "softwarelists", release); text.AppendLine("Processed Software database: " + release); } } } text.Append("OK"); return text.ToString(); } public static string[] ExtractBinSrcTxtLinks() { string[] result = new string[] { null, null, null }; string url = @"https://www.mamedev.org/release.php"; string[] links = Spludlow.Html.Web.DownloadLinks(url); foreach (string link in links) { string name = Path.GetFileName(link.ToLower().Trim()); if (name.Contains(".") == true && (name.StartsWith("mame") == true || name.StartsWith("whatsnew") == true)) { if (name.EndsWith(".txt") == false) { if (name.Contains("b") == true) { if (result[0] == null) result[0] = link; else throw new ApplicationException("Mame links more than 1 bin"); } if (name.Contains("s") == true) { if (result[1] == null) result[1] = link; else throw new ApplicationException("Mame links more than 1 src"); } } else { if (result[2] == null) result[2] = link; else throw new ApplicationException("Mame links more than 1 whatsnew"); } } } return result; } static string PrettyXml(string filename, XmlWriterSettings xmlWriterSettings) { XDocument xDoc = XDocument.Load(filename); StringBuilder output = new StringBuilder(); using (XmlWriter writer = XmlWriter.Create(output, xmlWriterSettings)) { xDoc.Save(writer); } return output.ToString(); } public static void CombineSoftwareList(string hashDirectory, string listsoftwareFilename, string targetFilename) { XmlWriterSettings xmlWriterSettings = new XmlWriterSettings(); xmlWriterSettings.IndentChars = "\t"; xmlWriterSettings.Indent = true; using (StreamWriter writer = new StreamWriter(targetFilename, false, Encoding.UTF8)) { string line; using (StreamReader reader = new StreamReader(listsoftwareFilename, Encoding.UTF8)) { while ((line = reader.ReadLine()) != null) { writer.WriteLine(line); if (line.StartsWith(""); } } private static void LoadDatabase(string xmlFilename, string databaseName, string topElementName, string release) { Spludlow.Data.IDAL database = MameDatabase.Server(); if (database.DatabaseExists(databaseName) == true) database.DatabaseDelete(databaseName); DataSet dataSet = Spludlow.Data.XML.ConvertRelational(xmlFilename, topElementName); TrimStrings(dataSet); DataSet schema = Spludlow.Data.ADO.Schema(dataSet, true); FixEmptyColumnsInSchema(schema); //Spludlow.Data.Database.AutoAddForeignKeys(schema); Spludlow.Data.Schemas.ImportDatabase(schema, dataSet, database, databaseName, null); database.ChangeDatabase(databaseName); WriteRelease(database, release); } public static void TrimStrings(DataSet dataSet) { Type type = typeof(string); foreach (DataTable table in dataSet.Tables) { foreach (DataColumn column in table.Columns) { if (column.DataType != type) continue; int ordinal = column.Ordinal; foreach (DataRow row in table.Rows) { if (row.IsNull(ordinal) == true) continue; string value = ((string)row[ordinal]).Trim(); if (value.Length == 0) row[ordinal] = DBNull.Value; else row[ordinal] = value; } } } } public static void FixEmptyColumnsInSchema(DataSet schema) { foreach (DataTable table in schema.Tables) { if (table.TableName.EndsWith("_Columns") == false) continue; foreach (DataRow row in table.Rows) { if ((int)row["MaxLength"] == -1 && (int)row["Longest"] == 0) { row["Longest"] = 1; Spludlow.Log.Warning("Fixing empty column length: " + table.TableName + ", " + (string)row["ColumnName"], table); } } } } private static bool RequireDatabaseUpdate(string databaseName, string release) { Spludlow.Data.IDAL database = MameDatabase.Server(); if (database.DatabaseExists(databaseName) == false) return true; database.ChangeDatabase(databaseName); if (database.TableExists("Versions") == false) return true; DataTable table = database.Select("SELECT * FROM Versions"); if (table.Rows.Count == 0) return true; if ((string)table.Rows[0]["Version"] != release) return true; return false; } private static void WriteRelease(Spludlow.Data.IDAL database, string release) { DataTable table = new DataTable("Versions"); table.Columns.Add("Version", typeof(string)); table.PrimaryKey = new DataColumn[] { table.Columns[0] }; table.Rows.Add(release); DataSet schema = Spludlow.Data.ADO.Schema(table, true); database.TableCreate(table.TableName, schema); database.Insert(table.Rows[0]); } } }