Fixes on debug printing

Lucidiot.Magellan.Extract/Lucidiot.Magellan.Extract.csproj

@@ -8,7 +8,7 @@
     <OutputType>Exe</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Lucidiot.Magellan.Extract</RootNamespace>
-    <AssemblyName>MagellanExtract</AssemblyName>
+    <AssemblyName>Lucidiot.Magellan.Extract</AssemblyName>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -53,4 +53,4 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-</Project>
+</Project>

Lucidiot.Magellan.Extract/Program.cs

@@ -4,6 +4,37 @@ using System.IO;
 using Lucidiot.Raima;
 
 namespace Lucidiot.Magellan.Extract {
+    /// <summary>
+    /// Helper to temporarily switch the console's colors to others, and then restore the original colors.
+    /// </summary>
+    /// <remarks>
+    /// This should be used with a <c>using</c> keyword: the <see cref="ColorSwitch.Dispose" /> method will restore the original colors.
+    /// </remarks>
+    internal class ColorSwitch : IDisposable {
+        public readonly ConsoleColor OriginalForeground;
+        public readonly ConsoleColor OriginalBackground;
+
+        public ColorSwitch() {
+            OriginalForeground = Console.ForegroundColor;
+            OriginalBackground = Console.BackgroundColor;
+        }
+
+        public ColorSwitch(ConsoleColor Foreground)
+            : this() {
+            Console.ForegroundColor = Foreground;
+        }
+
+        public ColorSwitch(ConsoleColor Foreground, ConsoleColor Background)
+            : this(Foreground) {
+            Console.BackgroundColor = Background;
+        }
+
+        public void Dispose() {
+            Console.ForegroundColor = OriginalForeground;
+            Console.BackgroundColor = OriginalBackground;
+        }
+    }
+
     class Program {
         static int Main(string[] args) {
             if (args.Length < 1) {
@@ -17,27 +48,56 @@ namespace Lucidiot.Magellan.Extract {
             return 0;
         }
 
+        static string HumanBytes(double bytes) {
+            char prefix;
+            if (bytes > 1E9) {
+                bytes /= 1E9;
+                prefix = 'G';
+            } else if (bytes > 1E6) {
+                bytes /= 1E6;
+                prefix = 'M';
+            } else if (bytes > 1E3) {
+                bytes /= 1E3;
+                prefix = 'K';
+            } else {
+                return String.Format("{0}B", bytes);
+            }
+            return String.Format("{0:F1}{1}B", bytes, prefix);
+        }
+
         static void DumpMGI(string MGIPath) {
             Console.WriteLine("Magellan Map archive contents - {0}", MGIPath);
             MapArchiveEntry[] entries;
             try {
                 entries = MapArchive.ReadHeader(MGIPath);
             } catch (Exception e) {
-                ConsoleColor current = Console.ForegroundColor;
+                using (new ColorSwitch(ConsoleColor.Red)) {
-                Console.ForegroundColor = ConsoleColor.Red;
+                    Console.WriteLine(e.ToString());
-                Console.WriteLine(e.ToString());
+                }
-                Console.ForegroundColor = current;
                 Console.WriteLine();
                 Console.WriteLine("--------------------------------------------------");
                 return;
             }
             List<MapArchiveEntry> databases = new List<MapArchiveEntry>();
             foreach (MapArchiveEntry entry in entries) {
-                Console.WriteLine("{0,-16} {1} bytes starting at offset {2}", entry.FullName, entry.Length, entry.Offset);
+                Console.WriteLine("{0,-16} {1} starting at offset {2}", entry.FullName, HumanBytes(entry.Length), entry.Offset);
                 if (entry.Extension.ToLowerInvariant() == "dbd") {
                     Console.WriteLine("  Looks like a Raima database, will attempt to dump its schema below.");
                     databases.Add(entry);
                 }
+
+                bool invalidChars = false;
+                foreach (char invalidChar in Path.GetInvalidFileNameChars()) {
+                    if (entry.FullName.Contains(invalidChar.ToString())) {
+                        invalidChars = true;
+                        break;
+                    }
+                }
+                if (invalidChars) {
+                    using (new ColorSwitch(ConsoleColor.Yellow)) {
+                        Console.WriteLine("  File name has invalid characters");
+                    }
+                }
             }
             Console.WriteLine();
             Console.WriteLine("--------------------------------------------------");
@@ -48,21 +108,22 @@ namespace Lucidiot.Magellan.Extract {
         }
 
         static void DumpDatabase(string MGIPath, string fileName) {
-            Console.WriteLine("Raima Database Manager Embedded V3.00 Database Schema Dump");
+            Console.WriteLine("Raima Database Manager Embedded Database Schema Dump");
             Console.WriteLine("Database {0} in archive {1}", fileName, MGIPath);
             DatabaseSchema ds;
             try {
                 ds = DatabaseSchema.Load(fileName, new MapArchiveStorage(MGIPath));
             } catch (Exception e) {
-                ConsoleColor current = Console.ForegroundColor;
+                using (new ColorSwitch(ConsoleColor.Red)) {
-                Console.ForegroundColor = ConsoleColor.Red;
+                    Console.WriteLine(e.ToString());
-                Console.WriteLine(e.ToString());
+                }
-                Console.ForegroundColor = current;
                 Console.WriteLine();
                 Console.WriteLine("--------------------------------------------------");
                 return;
             }
+            Console.WriteLine("Version: {0}", ds.Version);
             Console.WriteLine("PageSize: {0}", ds.PageSize);
+
             Console.WriteLine("--- FILE ENTRIES ---------------------------------");
             for (int i = 0; i < ds.FileEntries.Count; i++) {
                 FileEntry entry = ds.FileEntries[i];
@@ -80,6 +141,7 @@ namespace Lucidiot.Magellan.Extract {
             if (ds.FileEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- RECORD ENTRIES -------------------------------");
             for (int i = 0; i < ds.RecordEntries.Count; i++) {
                 RecordEntry entry = ds.RecordEntries[i];
@@ -93,10 +155,21 @@ namespace Lucidiot.Magellan.Extract {
                     Console.WriteLine("    Static");
                 if ((entry.Options & RecordEntryOptions.ContainsCompoundKey) == RecordEntryOptions.ContainsCompoundKey)
                     Console.WriteLine("    Contains a compound key");
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.FileEntryIndex < 0 || entry.FileEntryIndex + 1 > ds.FileEntries.Count)
+                        Console.WriteLine("    File #{0} does not exist", entry.FileEntryIndex);
+                    if (entry.FirstFieldEntryIndex < 0 || entry.FirstFieldEntryIndex + 1 > ds.FieldEntries.Count)
+                        Console.WriteLine("    Field #{0} does not exist", entry.FirstFieldEntryIndex);
+                    // TODO: No overlapping values when using the first index / count attributes
+                    if (entry.FirstFieldEntryIndex + entry.FieldEntryCount > ds.FieldEntries.Count)
+                        Console.WriteLine("    Uses more fields than available in the field entries list");
+                }
             }
             if (ds.RecordEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- FIELD ENTRIES --------------------------------");
             for (int i = 0; i < ds.FieldEntries.Count; i++) {
                 FieldEntry entry = ds.FieldEntries[i];
@@ -120,10 +193,54 @@ namespace Lucidiot.Magellan.Extract {
                     Console.WriteLine("    Part of a compound key");
                 if ((entry.Options & FieldEntryOptions.SortField) == FieldEntryOptions.SortField)
                     Console.WriteLine("    Part of a sort entry");
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.RecordEntryIndex < 0 || entry.RecordEntryIndex + 1 > ds.RecordEntries.Count)
+                        Console.WriteLine("    Record #{0} does not exist", entry.RecordEntryIndex);
+
+                    if (entry.Type == FieldType.CompoundKey) {
+                        if (entry.Offset < 0 || entry.Offset + 1 > ds.FieldEntries.Count)
+                            Console.WriteLine("    Field #{0} does not exist", entry.Offset);
+                        if (entry.Offset + entry.Length > ds.FieldEntries.Count)
+                            Console.WriteLine("    Uses more fields than available in the fields list");
+
+                        for (
+                            int j = Math.Max((short)0, entry.Offset);
+                            j < Math.Min(ds.FieldEntries.Count - Math.Max((short)0, entry.Offset), entry.Length);
+                            j++
+                        ) {
+                            FieldEntry keyMember = ds.FieldEntries[j];
+                            if ((entry.Options & FieldEntryOptions.CompoundKeyMember) != FieldEntryOptions.CompoundKeyMember)
+                                Console.WriteLine("    Field #{0} is not flagged as part of a compound key", j);
+                        }
+                    }
+
+                    if ((entry.Options & FieldEntryOptions.StructField) == FieldEntryOptions.StructField) {
+                        bool found = false;
+                        // Walk the field list backwards to try to find the related GroupedField
+                        for (int j = i - 1; j >= 0; j--) {
+                            FieldEntry previousField = ds.FieldEntries[j];
+                            if (previousField.Type == FieldType.GroupedField) {
+                                found = true;
+                                break;
+                            } else if ((entry.Options & FieldEntryOptions.StructField) != FieldEntryOptions.StructField) {
+                                /*
+                                 * A GroupedField should only be followed by fields flagged as part of a GroupedField;
+                                 * once a field that does not have this flag is found, it means the GroupedField ends.
+                                 * If we find a field that is neither a GroupedField nor flagged as part of one, stop.
+                                 */
+                                break;
+                            }
+                        }
+                        if (!found)
+                            Console.WriteLine("    GroupedField not found");
+                    }
+                }
             }
             if (ds.FieldEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- SET ENTRIES ----------------------------------");
             for (int i = 0; i < ds.SetEntries.Count; i++) {
                 SetEntry entry = ds.SetEntries[i];
@@ -133,32 +250,99 @@ namespace Lucidiot.Magellan.Extract {
                 Console.WriteLine("    Ordering: {0}", entry.Ordering);
                 if ((entry.Options & SetEntryOptions.Timestamped) == SetEntryOptions.Timestamped)
                     Console.WriteLine("    Timestamped");
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.RecordEntryIndex < 0 || entry.RecordEntryIndex + 1 > ds.RecordEntries.Count)
+                        Console.WriteLine("    Record #{0} does not exist", entry.RecordEntryIndex);
+                    if (entry.FirstMemberEntryIndex < 0 || entry.FirstMemberEntryIndex + 1 > ds.MemberEntries.Count)
+                        Console.WriteLine("    Member entry #{0} does not exist", entry.FirstMemberEntryIndex);
+                    // TODO: No overlapping values when using the first index / count attributes
+                    if (entry.FirstMemberEntryIndex + entry.MemberCount > ds.MemberEntries.Count)
+                        Console.WriteLine("    Uses more member entries than available in the member entries list");
+
+                    for (
+                        int j = Math.Max((short)0, entry.FirstMemberEntryIndex);
+                        j < Math.Min(ds.MemberEntries.Count - Math.Max((short)0, entry.FirstMemberEntryIndex), entry.MemberCount);
+                        j++
+                    ) {
+                        MemberEntry member = ds.MemberEntries[j];
+                        for (
+                            int k = Math.Max((short)0, member.FirstSortEntryIndex);
+                            k < Math.Min(ds.SortEntries.Count - Math.Max((short)0, member.FirstSortEntryIndex), member.SortEntryCount);
+                            k++
+                        ) {
+                            SortEntry sort = ds.SortEntries[k];
+                            if (sort.SetIndex != i)
+                                Console.WriteLine("    Sort entry #{0} in member entry #{1} references set #{2} instead", k, j, sort.SetIndex);
+                        }
+                    }
+                }
             }
             if (ds.SetEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- MEMBER ENTRIES -------------------------------");
             for (int i = 0; i < ds.MemberEntries.Count; i++) {
                 MemberEntry entry = ds.MemberEntries[i];
-                Console.WriteLine("  #{0}: member entry from record #{1}", i, entry.RecordIndex);
+                Console.WriteLine("  #{0}: member entry from record #{1}", i, entry.RecordEntryIndex);
-                Console.WriteLine("    Offset: ", entry.Offset);
+                Console.WriteLine("    Offset: {0}", entry.Offset);
                 Console.WriteLine("    {0} sort entries starting at #{1}", entry.SortEntryCount, entry.FirstSortEntryIndex);
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.RecordEntryIndex < 0 || entry.RecordEntryIndex + 1 > ds.RecordEntries.Count)
+                        Console.WriteLine("    Record #{0} does not exist", entry.RecordEntryIndex);
+                    if (entry.FirstSortEntryIndex < 0 || entry.FirstSortEntryIndex + 1 > ds.SortEntries.Count)
+                        Console.WriteLine("    Sort entry #{0} does not exist", entry.FirstSortEntryIndex);
+                    // TODO: No overlapping values when using the first index / count attributes
+                    if (entry.FirstSortEntryIndex + entry.SortEntryCount > ds.SortEntries.Count)
+                        Console.WriteLine("    Uses more sort entries than available in the sort entries list");
+                }
             }
             if (ds.MemberEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- SORT ENTRIES ---------------------------------");
             for (int i = 0; i < ds.SortEntries.Count; i++) {
                 SortEntry entry = ds.SortEntries[i];
                 Console.WriteLine("  #{0}: set #{1} - field #{2}", i, entry.SetIndex, entry.FieldIndex);
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.SetIndex < 0 || entry.SetIndex + 1 > ds.SetEntries.Count)
+                        Console.WriteLine("    Set #{0} does not exist", entry.SetIndex);
+                    if (entry.FieldIndex < 0 || entry.FieldIndex + 1 > ds.FieldEntries.Count)
+                        Console.WriteLine("    Field #{0} does not exist", entry.FieldIndex);
+                    else if ((ds.FieldEntries[entry.FieldIndex].Options & FieldEntryOptions.SortField) != FieldEntryOptions.SortField)
+                        Console.WriteLine("    Field #{0} is not flagged as part of a sort entry", entry.FieldIndex);
+                }
             }
             if (ds.SortEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--- KEY ENTRIES ----------------------------------");
+            for (int i = 0; i < ds.KeyEntries.Count; i++) {
+                KeyEntry entry = ds.KeyEntries[i];
+                Console.WriteLine("  #{0}: field #{1} on compound key field #{2}", i, entry.FieldIndex, entry.KeyFieldIndex);
+                Console.WriteLine("    Offset: {0}", entry.Offset);
+                Console.WriteLine("    Ordering: {0}", entry.Ordering);
+
+                using (new ColorSwitch(ConsoleColor.Yellow)) {
+                    if (entry.KeyFieldIndex < 0 || entry.KeyFieldIndex + 1 > ds.FieldEntries.Count)
+                        Console.WriteLine("    Key field #{0} does not exist", entry.KeyFieldIndex);
+                    else if (ds.FieldEntries[entry.KeyFieldIndex].Type != FieldType.CompoundKey)
+                        Console.WriteLine("    Key field #{0} is not a compound key field", entry.KeyFieldIndex);
+                    if (entry.FieldIndex < 0 || entry.FieldIndex + 1 > ds.FieldEntries.Count)
+                        Console.WriteLine("    Field #{0} does not exist", entry.FieldIndex);
+                    else if ((ds.FieldEntries[entry.FieldIndex].Options & FieldEntryOptions.CompoundKeyMember) != FieldEntryOptions.CompoundKeyMember)
+                        Console.WriteLine("    Field #{0} is not flagged as part of a compound key", entry.FieldIndex);
+                }
+            }
             if (ds.KeyEntries.Count < 1)
                 Console.WriteLine("  No entries found.");
             Console.WriteLine();
+
             Console.WriteLine("--------------------------------------------------");
         }
     }

Lucidiot.Magellan/Lucidiot.Magellan.csproj

@@ -8,7 +8,7 @@
     <OutputType>Library</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Lucidiot.Magellan</RootNamespace>
-    <AssemblyName>MagellanLibrary</AssemblyName>
+    <AssemblyName>Lucidiot.Magellan</AssemblyName>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -59,4 +59,4 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-</Project>
+</Project>

Lucidiot.Raima/DatabaseSchema.cs

@@ -9,16 +9,30 @@ namespace Lucidiot.Raima {
         /// <summary>
         /// Version 3.00, with a <c>V3.00 + 0x1A</c> file magic.
         /// </summary>
-        V300
+        V300,
+
+        /// <summary>
+        /// Version 3.10, with a <c>V3.10 + 0x1A</c> file magic.
+        /// </summary>
+        V310
     }
 
     public class DatabaseSchema {
         private short _pageSize = 512;
         public short PageSize { get { return _pageSize; } }
 
-        public const DatabaseVersion Version = DatabaseVersion.V300;
+        public DatabaseVersion Version = DatabaseVersion.V300;
 
-        public const string Magic = "V3.00\x1a";
+        public const int MagicNumberLength = 6;
+
+        public Dictionary<string, DatabaseVersion> MagicNumbers {
+            get {
+                Dictionary<string, DatabaseVersion> numbers = new Dictionary<string,DatabaseVersion>();
+                numbers["V3.00\x1a"] = DatabaseVersion.V300;
+                numbers["V3.10\x1a"] = DatabaseVersion.V310;
+                return numbers;
+            }
+        }
 
         public string DatabaseFileName;
 
@@ -71,6 +85,10 @@ namespace Lucidiot.Raima {
             this.StorageMethod = storage;
         }
 
+        public DatabaseSchema(string fileName, IDatabaseStorage storage, DatabaseVersion version) : this(fileName, storage) {
+            this.Version = version;
+        }
+
         public static DatabaseSchema Load(string fileName, IDatabaseStorage storage) {
             if (!storage.CanRead)
                 throw new NotSupportedException("A readable storage method is required to load an existing database.");
@@ -82,9 +100,10 @@ namespace Lucidiot.Raima {
         }
 
         private void Parse(BinaryReader br) {
-            string foundMagic = new String(Encoding.ASCII.GetChars(br.ReadBytes(Magic.Length)));
+            string foundMagic = new String(Encoding.ASCII.GetChars(br.ReadBytes(MagicNumberLength)));
-            if (foundMagic != Magic)
+            if (!MagicNumbers.ContainsKey(foundMagic))
                 throw new InvalidDataException("Unexpected file magic: " + foundMagic);
+            Version = MagicNumbers[foundMagic];
             short pageSize = br.ReadInt16();
             if (pageSize % 512 != 0)
                 throw new ArgumentOutOfRangeException("Page size must be a multiple of 512 bytes.");

Lucidiot.Raima/Lucidiot.Raima.csproj

@@ -8,7 +8,7 @@
     <OutputType>Library</OutputType>
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Lucidiot.Raima</RootNamespace>
-    <AssemblyName>RaimaLibrary</AssemblyName>
+    <AssemblyName>Lucidiot.Raima</AssemblyName>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>

Lucidiot.Raima/MemberEntry.cs

@@ -4,7 +4,7 @@ namespace Lucidiot.Raima {
     [StructLayout(LayoutKind.Explicit, Size = 8)]
     public struct MemberEntry {
         [FieldOffset(0)]
-        public short RecordIndex;
+        public short RecordEntryIndex;
 
         [FieldOffset(2)]
         public short Offset;

Lucidiot.Raima/SetEntry.cs

@@ -2,12 +2,12 @@ using System;
 using System.Runtime.InteropServices;
 
 namespace Lucidiot.Raima {
-    public enum SetOrdering : byte {
+    public enum SetOrdering : ushort {
-        Ascending = (byte)'a',
+        Ascending = (ushort)'a',
-        Descending = (byte)'d',
+        Descending = (ushort)'d',
-        First = (byte)'f',
+        First = (ushort)'f',
-        Last = (byte)'l',
+        Last = (ushort)'l',
-        None = (byte)'n'
+        None = (ushort)'n'
     }
 
     [Flags]
@@ -16,24 +16,24 @@ namespace Lucidiot.Raima {
         Timestamped = 0x0001
     }
 
-    [StructLayout(LayoutKind.Explicit, Size = 11)]
+    [StructLayout(LayoutKind.Explicit, Size = 12)]
     public struct SetEntry {
         [FieldOffset(0)]
         public SetOrdering Ordering;
 
-        [FieldOffset(1)]
+        [FieldOffset(2)]
         public short RecordEntryIndex;
 
-        [FieldOffset(3)]
+        [FieldOffset(4)]
         public short Offset;
 
-        [FieldOffset(5)]
+        [FieldOffset(6)]
         public short FirstMemberEntryIndex;
 
-        [FieldOffset(7)]
+        [FieldOffset(8)]
         public short MemberCount;
 
-        [FieldOffset(9)]
+        [FieldOffset(10)]
         public SetEntryOptions Options;
     }
 }