AUP3: Added AUP3 importer and improved progress dialogs

cmake-proxies/sqlite/CMakeLists.txt

@@ -19,10 +19,23 @@ list( APPEND INCLUDES
 
 list( APPEND DEFINES
    PRIVATE
-      SQLITE_ENABLE_SNAPSHOT=1
+      #
-      SQLITE_DQS=0
+      # Connection based settings. Persistent settings are done in the
-#     SQLITE_DEFAULT_PAGE_SIZE=4096
+      # schema.
+      #
+      SQLITE_DEFAULT_LOCKING_MODE=1
+      SQLITE_DEFAULT_WAL_AUTOCHECKPOINT=100
+      SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=50000000
+
+      # Can't be set after a WAL mode database is initialized, so change
+      # the default here to ensure all project files get the same page 
+      # size.
       SQLITE_DEFAULT_PAGE_SIZE=65536
+
+      #
+      # Recommended in SQLite docs
+      #
+      SQLITE_DQS=0
       SQLITE_DEFAULT_MEMSTATUS=0
       SQLITE_DEFAULT_WAL_SYNCHRONOUS=1
       SQLITE_LIKE_DOESNT_MATCH_BLOBS

cmake-proxies/wxWidgets/CMakeLists.txt

@@ -18,7 +18,7 @@ if( ${_OPT}use_wxwidgets STREQUAL "system" )
 
    # Specify all of the components we'll need since "html" and "qa" aren't
    # included in the default list
-   find_package(wxWidgets REQUIRED COMPONENTS adv base core html net qa)
+   find_package(wxWidgets REQUIRED COMPONENTS adv base core html net qa xml)
 
    unset( BUILD_SHARED_LIBS )
 endif()

src/ProjectFileIO.cpp

@@ -14,6 +14,8 @@ Paul Licameli split from AudacityProject.cpp
 #include <sqlite3.h>
 #include <wx/crt.h>
 #include <wx/frame.h>
+#include <wx/sstream.h>
+#include <wx/xml/xml.h>
 
 #include <thread>
 
@@ -47,10 +49,13 @@ static const int ProjectFileVersion = 1;
 // to sampleblocks.
 
 static const char *ProjectFileSchema =
+   // These are persistent and not connection based
+   //
+   // See the CMakeList.txt for the SQLite lib for more
+   // settings.
    "PRAGMA <dbname>.application_id = %d;"
    "PRAGMA <dbname>.user_version = %d;"
    "PRAGMA <dbname>.journal_mode = WAL;"
-   "PRAGMA <dbname>.locking_mode = EXCLUSIVE;"
    ""
    // project is a binary representation of an XML file.
    // it's in binary for speed.
@@ -375,6 +380,13 @@ sqlite3 *ProjectFileIO::OpenDB(FilePath fileName)
       return nullptr;
    }
 
+   // These are here for easier research. Permanent settings should be done
+   // in the CMake list for SQLite.
+#if 1
+//   rc = sqlite3_exec(mDB, "PRAGMA wal_autocheckpoint=1000;", nullptr, nullptr, nullptr);
+//   rc = sqlite3_exec(mDB, "PRAGMA journal_size_limit=1000000000;", nullptr, nullptr, nullptr);
+#endif
+
    if (!CheckVersion())
    {
       CloseDB();
@@ -497,7 +509,7 @@ int ProjectFileIO::ExecCallback(void *data, int cols, char **vals, char **names)
    return parms->func(parms->result, cols, vals, names);
 }
 
-int ProjectFileIO::Exec(const char *query, ExecCB callback, wxString *result)
+int ProjectFileIO::Exec(const char *query, ExecCB callback, std::vector<wxString> *result)
 {
    char *errmsg = nullptr;
    ExecParm ep = {callback, result};
@@ -516,15 +528,15 @@ int ProjectFileIO::Exec(const char *query, ExecCB callback, wxString *result)
    return rc;
 }
 
-bool ProjectFileIO::GetValue(const char *sql, wxString &result)
+bool ProjectFileIO::GetValues(const char *sql, std::vector<wxString> &result)
 {
    result.clear();
 
-   auto getresult = [](wxString *result, int cols, char **vals, char **names)
+   auto getresult = [](std::vector<wxString> *result, int cols, char **vals, char **names)
    {
       if (cols == 1 && vals[0])
       {
-         result->assign(vals[0]);
+         result->push_back(vals[0]);
          return SQLITE_OK;
       }
 
@@ -540,6 +552,24 @@ bool ProjectFileIO::GetValue(const char *sql, wxString &result)
    return true;
 }
 
+bool ProjectFileIO::GetValue(const char *sql, wxString &result)
+{
+   result.clear();
+
+   std::vector<wxString> holder;
+   if (!GetValues(sql, holder))
+   {
+      return false;
+   }
+
+   if (holder.size())
+   {
+      result.assign(holder[0]);
+   }
+
+   return true;
+}
+
 bool ProjectFileIO::GetBlob(const char *sql, wxMemoryBuffer &buffer)
 {
    auto db = DB();
@@ -734,7 +764,9 @@ void ProjectFileIO::UpdateCallback(void *data, int operation, char const *dbname
    cb(operation, dbname, table, rowid);
 }
 
-sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false */)
+sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath,
+                               const TranslatableString &msg,
+                               bool prune /* = false */)
 {
    // Get access to the active tracklist
    auto pProject = mpProject.lock();
@@ -744,21 +776,42 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false *
    }
    auto &tracklist = TrackList::Get(*pProject);
 
-   // Collect all active blockids
    BlockIDs blockids;
-   for (auto wt : tracklist.Any<const WaveTrack>())
+
+   // Collect all active blockids
+   if (prune)
    {
-      // Scan all clips within current track
+      for (auto wt : tracklist.Any<const WaveTrack>())
-      for (const auto &clip : wt->GetAllClips())
       {
-         // Scan all blockfiles within current clip
+         // Scan all clips within current track
-         auto blocks = clip->GetSequenceBlockArray();
+         for (const auto &clip : wt->GetAllClips())
-         for (const auto &block : *blocks)
          {
-            blockids.insert(block.sb->GetBlockID());
+            // Scan all blockfiles within current clip
+            auto blocks = clip->GetSequenceBlockArray();
+            for (const auto &block : *blocks)
+            {
+               blockids.insert(block.sb->GetBlockID());
+            }
          }
       }
    }
+   // Collect ALL blockids
+   else
+   {
+      std::vector<wxString> holder;
+      if (!GetValues("SELECT blockid FROM sampleblocks;", holder))
+      {
+         return nullptr;
+      }
+
+      for (auto block : holder)
+      {
+         SampleBlockID blockid;
+         block.ToLongLong(&blockid);
+
+         blockids.insert(blockid);
+      }
+   }
 
    auto db = DB();
    sqlite3 *destdb = nullptr;
@@ -772,12 +825,6 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false *
       // Detach the destination database, whether it was successfully attached or not
       sqlite3_exec(db, "DETACH DATABASE dest;", nullptr, nullptr, nullptr);
 
-      // Remove our function, whether it was successfully defined or not.
-      sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, nullptr, nullptr, nullptr);
-
-      // Remove the update hook, whether it was successfully defined or not.
-      sqlite3_update_hook(db, nullptr, nullptr);
-
       if (!success)
       {
          sqlite3_close(destdb);
@@ -786,16 +833,6 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false *
       }
    });
 
-   // Add the function used to verify each rows blockid against the set of active blockids
-   rc = sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, &blockids, InSet, nullptr, nullptr);
-   if (rc != SQLITE_OK)
-   {
-      SetDBError(
-         XO("Unable to add 'inset' function")
-      );
-      return nullptr;
-   }
-
    // Attach the destination database 
    wxString sql;
    sql.Printf("ATTACH DATABASE '%s' AS dest;", destpath);
@@ -816,91 +853,65 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false *
       return nullptr;
    }
 
-   // This whole progress thing is a mess...gotta work on it more
+   rc = sqlite3_exec(db,
+                     "INSERT INTO dest.tags SELECT * FROM main.tags;",
+                     nullptr,
+                     nullptr,
+                     nullptr);
+   if (rc != SQLITE_OK)
+   {
+      SetDBError(
+         XO("Failed to copy tags")
+      );
+
+      return nullptr;
+   }
 
    {
       /* i18n-hint: This title appears on a dialog that indicates the progress
          in doing something.*/
-      ProgressDialog progress(XO("Progress"), XO("Saving project"));
+      ProgressDialog progress(XO("Progress"), msg, pdlgHideStopButton);
+      ProgressResult result = ProgressResult::Success;
 
       wxLongLong_t count = 0;
       wxLongLong_t total = blockids.size();
 
-      UpdateCB update = [&](int operation, char const *dbname, char const *table, long long rowid)
+      for (auto blockid : blockids)
-      {
-         count++;
-      };
-      sqlite3_update_hook(db, UpdateCallback, &update);
-
-      // The first version is a better solution, but after the rows get copied
-      // there's a large delay when processing large files.  This is due to the
-      // checkpointing of the WAL to the DB.  There might be something we can
-      // do with the WAL hook to throw up another progress dialog while it does
-      // the checkpointing, but the second version (sort of) get's around it.
-      //
-      // The second version does the copy in a separate thread.  This allows
-      // us to keep the progress dialog active and elapsed time updating while
-      // the checkpointing is occurring.
-      //
-      // One possible thing to try is to only copy a smallish number of rows
-      // at a time to spread out the time required to checkpoint so the user
-      // doesn't notice it.
-      //
-      // Neither are ideal and more research is required.
-#if 0
-      rc = sqlite3_exec(db,
-                        "INSERT INTO dest.tags SELECT * FROM main.tags;",
-                        nullptr,
-                        nullptr,
-                        nullptr);
-      if (rc == SQLITE_OK)
       {
          wxString sql;
-         sql.Printf("INSERT INTO dest.sampleblocks SELECT * FROM main.sampleblocks%s;",
+         sql.Printf("INSERT INTO dest.sampleblocks"
-                    prune ? " WHERE inset(blockid)" : "");
+                     "  SELECT * FROM main.sampleblocks"
+                     "  WHERE blockid = %lld",
+                     blockid);
 
          rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
+         if (rc != SQLITE_OK)
+         {
+            SetDBError(
+               XO("Failed to copy project file")
+            );
+
+            return nullptr;
+         }
+/*
+         rc = sqlite3_wal_checkpoint_v2(db, "dest", SQLITE_CHECKPOINT_FULL, nullptr, nullptr);
+         if (rc != SQLITE_OK)
+         {
+            SetDBError(
+               XO("Failed to copy project file")
+            );
+
+            return nullptr;
+         }
+*/
+         result = progress.Update(++count, total);
+         if (result != ProgressResult::Success)
+         {
+            // Note that we're not setting success, so the finally
+            // block above will take care of cleaning up
+            return nullptr;
+         }
       }
-#else
-      bool done = false;
-      auto task = [&]()
-      {
-         rc = sqlite3_exec(db,
-                           "INSERT INTO dest.tags SELECT * FROM main.tags;",
-                           nullptr,
-                           nullptr,
-                           nullptr);
-         if (rc == SQLITE_OK)
-         {
-            wxString sql;
-            sql.Printf("INSERT INTO dest.sampleblocks SELECT * FROM main.sampleblocks%s;",
-                       prune ? " WHERE inset(blockid)" : "");
-
-            rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
-
-            sqlite3_wal_checkpoint_v2(db, "dest", SQLITE_CHECKPOINT_FULL, nullptr, nullptr);
-         }
-
-         done = true;
-      };
-
-      std::thread t = std::thread(task);
-      while (!done)
-      {
-         TranslatableString msg;
-         if (count < total)
-         {
-            msg = XO("Processing %lld of %lld sample blocks").Format(count, total);
-         }
-         else
-         {
-            msg = XO("Flushing journal to project file");
-         }
-         progress.Update(count, total, msg);
-         wxMilliSleep(100);
-      }
-      t.join();
-#endif
    }
 
    // Copy failed
@@ -930,44 +941,8 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath, bool prune /* = false *
    return destdb;
 }
 
-namespace {
-   unsigned long long
-   CalculateUsage(const TrackList &tracks)
-   {
-      std::set<long long> seen;
 
-      unsigned long long result = 0;
+unsigned long long ProjectFileIO::CalculateUsage()
-
-      //TIMER_START( "CalculateSpaceUsage", space_calc );
-      for (auto wt : tracks.Any< const WaveTrack >())
-      {
-         // Scan all clips within current track
-         for(const auto &clip : wt->GetAllClips())
-         {
-            // Scan all blockfiles within current clip
-            auto blocks = clip->GetSequenceBlockArray();
-            for (const auto &block : *blocks)
-            {
-               const auto &sb = block.sb;
-
-               // Accumulate space used by the blockid if the blocckid was not
-               // yet seen
-               if ( seen.count( sb->GetBlockID() ) == 0 )
-               {
-                  result += sb->GetSpaceUsage();
-               }
-
-               // Add file to current set
-               seen.insert( sb->GetBlockID() );
-            }
-         }
-      }
-
-      return result;
-   }
-}
-
-bool ProjectFileIO::Vacuum()
 {
    // Collect all active block usage
    auto pProject = mpProject.lock();
@@ -976,7 +951,42 @@ bool ProjectFileIO::Vacuum()
       return false;
    }
    auto &tracklist = TrackList::Get(*pProject);
-   double used = (double) CalculateUsage(tracklist);
+
+   std::set<long long> seen;
+
+   unsigned long long result = 0;
+
+   //TIMER_START( "CalculateSpaceUsage", space_calc );
+   for (auto wt : tracklist.Any<const WaveTrack>())
+   {
+      // Scan all clips within current track
+      for (const auto &clip : wt->GetAllClips())
+      {
+         // Scan all blockfiles within current clip
+         auto blocks = clip->GetSequenceBlockArray();
+         for (const auto &block : *blocks)
+         {
+            const auto &sb = block.sb;
+
+            // Accumulate space used by the blockid if the blocckid was not
+            // yet seen
+            if (seen.count( sb->GetBlockID()) == 0)
+            {
+               result += sb->GetSpaceUsage();
+            }
+
+            // Add file to current set
+            seen.insert(sb->GetBlockID());
+         }
+      }
+   }
+
+   return result;
+}
+
+bool ProjectFileIO::Vacuum()
+{
+   double used = (double) CalculateUsage();
 
    // Collect total usage
    wxString result;
@@ -987,8 +997,8 @@ bool ProjectFileIO::Vacuum()
    double total;
    result.ToDouble(&total);
 
-   wxLogDebug(wxT("used = %f total = %f %f\n"), used, total, (used / total) * 100.0);
+   wxLogDebug(wxT("used = %f total = %f %f\n"), used, total, used / total);
-   if (((used / total) * 100.0) > 20.0)
+   if (used / total > 0.80)
    {
       wxLogDebug(wxT("not vacuuming"));
       return true;
@@ -1009,6 +1019,9 @@ bool ProjectFileIO::Vacuum()
       return false;
    }
 
+   // Shouldn't need to do this, but doesn't hurt.
+   wxRemoveFile(tempName);
+
    // If we can't rename the original to temporary, backout
    if (!wxRenameFile(origName, tempName))
    {
@@ -1032,17 +1045,22 @@ bool ProjectFileIO::Vacuum()
    }
 
    // Copy the original database to a new database while pruning unused sample blocks
-   auto newDB = CopyTo(origName, true);
+   auto newDB = CopyTo(origName, XO("Compacting project"), true);
 
    // If the copy failed or we aren't able to write the project doc, backout
    if (!newDB || !WriteDoc("project", doc, newDB))
    {
+      // Close the new and original DBs
       sqlite3_close(newDB);
+      CloseDB();
 
+      // AUD3 warn user somehow
       wxRemoveFile(origName);
 
+      // AUD3 warn user somehow
       wxRenameFile(tempName, origName);
 
+      // Reopen the original DB
       OpenDB(origName);
 
       return false;
@@ -1456,23 +1474,306 @@ bool ProjectFileIO::WriteDoc(const char *table,
    return true;
 }
 
+// Importing an AUP3 project into an AUP3 project is a bit different than
+// normal importing since we need to copy data from one DB to the other
+// while adjusting the sample block IDs to represent the newly assigned
+// IDs.
+bool ProjectFileIO::ImportProject(const FilePath &fileName)
+{
+   bool success = false;
+   bool restore = true;
+   int rc;
+
+   // We need either the autosave or project docs from the inbound AUP3
+   wxString project;
+   {
+      sqlite3 *indb = nullptr;
+
+      // Make sure we always return to the active project file
+      auto cleanup = finally([&]
+      {
+         if (indb)
+         {
+            sqlite3_close(indb);
+         }
+
+         RestoreConnection();
+      });
+      SaveConnection();
+
+      // Would be nice if we could open it read-only, but SQLITE_IOERR is
+      // returned when doing so.
+      rc = sqlite3_open(fileName, &indb);
+      if (rc != SQLITE_OK)
+      {
+         SetDBError(
+            XO("Unable to open project file:\n\n%s").Format(fileName)
+         );
+
+         return false;
+      }
+
+      // The inbound project file becomes the active project file
+      UseConnection(indb, fileName);
+
+      BlockIDs blockids;
+      wxMemoryBuffer buffer;
+
+      // Get the autosave doc, if any
+      if (!GetBlob("SELECT dict || doc FROM project WHERE id = 1;", buffer))
+      {
+         // Error already set
+         return false;
+      }
+ 
+      // If we didn't have an autosave doc, load the project doc instead
+      if (buffer.GetDataLen() == 0)
+      {
+         if (!GetBlob("SELECT dict || doc FROM autosave WHERE id = 1;", buffer))
+         {
+            // Error already set
+            return false;
+         }
+
+         // Missing both the autosave and project docs...this shouldn't happen!!!
+         if (buffer.GetDataLen() > 0)
+         {
+            SetError(XO("Unable to load project or autosave documents"));
+            return false;
+         }
+      }
+
+      // Decode it while capturing the associated sample blockids
+      project = ProjectSerializer::Decode(buffer, blockids);
+      if (project.size() == 0)
+      {
+         SetError(XO("Unable to decode project document"));
+
+         return false;
+      }
+   }
+  
+   // Parse the project doc
+   wxStringInputStream in(project);
+   wxXmlDocument doc;
+   if (!doc.Load(in))
+   {
+      return false;
+   }
+
+   // Get the root ("project") node
+   wxXmlNode *root = doc.GetRoot();
+   wxASSERT(root->GetName().IsSameAs(wxT("project")));
+
+   // Soft delete all non-essential attributes to prevent updating the active
+   // project. This takes advantage of the knowledge that when a project is
+   // parsed, unrecognized attributes are simply ignored.
+   //
+   // This is necessary because we don't want any of the active project settings
+   // to be modified by the inbound project.
+   for (wxXmlAttribute *attr = root->GetAttributes(); attr; attr = attr->GetNext())
+   {
+      wxString name = attr->GetName();
+      if (!name.IsSameAs(wxT("version")) && !name.IsSameAs(wxT("audacityversion")))
+      {
+         attr->SetName(name + wxT("_deleted"));
+      }
+   }
+
+   // Recursively find and collect all waveblock nodes
+   std::vector<wxXmlNode *> blocknodes;
+   std::function<void(wxXmlNode *)> findblocks = [&](wxXmlNode *node)
+   {
+      while (node)
+      {
+         if (node->GetName().IsSameAs(wxT("waveblock")))
+         {
+            blocknodes.push_back(node);
+         }
+         else
+         {
+            findblocks(node->GetChildren());
+         }
+
+         node = node->GetNext();
+      }
+   };
+
+   // Find all waveblocks in all wavetracks
+   for (wxXmlNode *node = doc.GetRoot()->GetChildren(); node; node = node->GetNext())
+   {
+      if (node->GetName().IsSameAs(wxT("wavetrack")))
+      {
+         findblocks(node->GetChildren());
+      }
+   }
+
+   {
+      // Get access to the current project file
+      auto db = DB();
+
+      // Make sure the inbound project file gets detached
+      auto cleanup = finally([&]
+      {
+         // Detach the inbound project file, whether it was successfully attached or not
+         sqlite3_exec(db, "DETACH DATABASE dest;", nullptr, nullptr, nullptr);
+      });
+
+      // Attach the inbound project file
+      wxString sql;
+      sql.Printf("ATTACH DATABASE '%s' AS inbound;", fileName);
+
+      rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
+      if (rc != SQLITE_OK)
+      {
+         SetDBError(
+            XO("Unable to attach %s project file").Format(fileName)
+         );
+
+         return false;
+      }
+
+      /* i18n-hint: This title appears on a dialog that indicates the progress
+         in doing something.*/
+      ProgressDialog progress(XO("Progress"), XO("Importing project"));
+      ProgressResult result = ProgressResult::Success;
+
+      wxLongLong_t count = 0;
+      wxLongLong_t total = blocknodes.size();
+
+      // Copy all the sample blocks from the inbound project file into
+      // the active one, while remembering which were copied.
+      std::vector<SampleBlockID> copied;
+      for (auto node : blocknodes)
+      {
+         // If the user cancelled the import or the import failed for some other reason
+         // make sure to back out the blocks copied to the active project file
+         auto backout = finally([&]
+         {
+            if (result == ProgressResult::Cancelled || result == ProgressResult::Failed)
+            {
+               for (auto blockid : copied)
+               {
+                  wxString sql;
+                  sql.Printf("DELETE FROM main.sampleblocks WHERE blockid = %lld", blockid);
+
+                  rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
+                  if (rc != SQLITE_OK)
+                  {
+                     // This is non-fatal...it'll just get cleaned up the next
+                     // time the project is opened.
+                     SetDBError(
+                        XO("Failed to delete block while cancelling import")
+                     );
+                  }
+               }
+            }
+         });
+
+         // Find the blockid attribute...it should always be there
+         wxXmlAttribute *attr = node->GetAttributes();
+         while (attr && !attr->GetName().IsSameAs(wxT("blockid")))
+         {
+            attr = attr->GetNext();
+         }
+         wxASSERT(attr != nullptr);
+
+         // And get the blockid
+         SampleBlockID blockid;
+         attr->GetValue().ToLongLong(&blockid);
+
+         // Copy the sample block from the inbound project to the active project.
+         // All columns other than the blockid column gets copied.
+         wxString columns(wxT("sampleformat, summin, summax, sumrms, summary256, summary64k, samples"));
+         wxString sql;
+         sql.Printf("INSERT INTO main.sampleblocks (%s)"
+                    "   SELECT %s"
+                    "   FROM inbound.sampleblocks"
+                    "   WHERE blockid = %lld",
+                    columns,
+                    columns,
+                    blockid);
+         rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
+         if (rc != SQLITE_OK)
+         {
+            SetDBError(
+               XO("Failed to import sample block")
+            );
+
+            result = ProgressResult::Failed;
+
+            break;
+         }
+
+         // Replace the original blockid with the new one
+         attr->SetValue(wxString::Format(wxT("%lld"), sqlite3_last_insert_rowid(db)));
+
+         // Remember that we copied this node in case the user cancels
+         result = progress.Update(++count, total);
+         if (result != ProgressResult::Success)
+         {
+            break;
+         }
+      }
+
+      // Bail if the import was cancelled or failed. If the user stopped the
+      // import or it completed, then we continue on.
+      if (result == ProgressResult::Cancelled || result == ProgressResult::Failed)
+      {
+         return false;
+      }
+
+      // Copy over tags...likely to produce duplicates...needs work once used
+      rc = sqlite3_exec(db,
+                        "INSERT INTO main.tags SELECT * FROM inbound.tags;",
+                        nullptr,
+                        nullptr,
+                        nullptr);
+      if (rc != SQLITE_OK)
+      {
+         SetDBError(
+            XO("Failed to import tags")
+         );
+
+         return false;
+      }
+
+      // We're all done with the inbound project file...finally() will detach it
+   }
+
+   // Recreate the project doc with the revisions we've made above
+   wxStringOutputStream output;
+   doc.Save(output);
+
+   // Now load the document as normal
+   XMLFileReader xmlFile;
+   if (!xmlFile.ParseString(this, output.GetString()))
+   {
+      SetError(
+         XO("Unable to parse project information.")
+      );
+      mLibraryError = xmlFile.GetErrorStr();
+
+      return false;
+   }
+
+   return true;
+}
+
 bool ProjectFileIO::LoadProject(const FilePath &fileName)
 {
    bool success = false;
 
-   // OpenDB() will change mFileName if the file opens/verifies successfully,
-   // so we must set it back to what it was if any errors are encountered here.
-   wxString oldFilename = mFileName;
-
    auto cleanup = finally([&]
    {
       if (!success)
       {
-         CloseDB();
+         RestoreConnection();
-         SetFileName(oldFilename);
       }
    });
 
+   SaveConnection();
+
    // Open the project file
    if (!OpenDB(fileName))
    {
@@ -1480,39 +1781,45 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName)
    }
 
    BlockIDs blockids;
-   wxString autosave;
    wxString project;
    wxMemoryBuffer buffer;
+   bool usedAutosave = true;
 
    // Get the autosave doc, if any
-   if (!GetBlob("SELECT dict || doc FROM project WHERE id = 1;", buffer))
-   {
-      // Error already set
-      return false;
-   }
-   if (buffer.GetDataLen() > 0)
-   {
-      project = ProjectSerializer::Decode(buffer, blockids);
-   }
-
    if (!GetBlob("SELECT dict || doc FROM autosave WHERE id = 1;", buffer))
    {
       // Error already set
       return false;
    }
-   if (buffer.GetDataLen() > 0)
+ 
+   // If we didn't have an autosave doc, load the project doc instead
+   if (buffer.GetDataLen() == 0)
    {
-      autosave = ProjectSerializer::Decode(buffer, blockids);
+      usedAutosave = false;
+
+      if (!GetBlob("SELECT dict || doc FROM project WHERE id = 1;", buffer))
+      {
+         // Error already set
+         return false;
+      }
+
+      if (buffer.GetDataLen() == 0)
+      {
+         SetError(XO("Unable to load project or autosave documents"));
+         return false;
+      }
    }
 
-   // Should this be an error???
+   // Decode it while capturing the associated sample blockids
-   if (project.empty() && autosave.empty())
+   project = ProjectSerializer::Decode(buffer, blockids);
+   if (project.empty())
    {
-      SetError(XO("Unable to load project or autosave documents"));
+      SetError(XO("Unable to decode project document"));
+
       return false;
    }
 
-   // Check for orphans blocks...set mRecovered if any deleted
+   // Check for orphans blocks...sets mRecovered if any were deleted
    if (blockids.size() > 0)
    {
       if (!CheckForOrphans(blockids))
@@ -1523,7 +1830,8 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName)
 
    XMLFileReader xmlFile;
 
-   success = xmlFile.ParseString(this, autosave.empty() ? project : autosave);
+   // Load 'er up
+   success = xmlFile.ParseString(this, project);
    if (!success)
    {
       SetError(
@@ -1534,7 +1842,7 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName)
    }
 
    // Remember if we used autosave or not
-   if (!autosave.empty())
+   if (usedAutosave)
    {
       mRecovered = true;
    }
@@ -1547,7 +1855,7 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName)
 
    // A previously saved project will have a document in the project table, so
    // we use that knowledge to determine if this file is an unsaved/temporary
-   // file or not
+   // file or a permanent project file
    wxString result;
    if (!GetValue("SELECT Count(*) FROM project;", result))
    {
@@ -1558,6 +1866,8 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName)
 
    SetFileName(fileName);
 
+   DiscardConnection();
+
    return true;
 }
 
@@ -1602,11 +1912,11 @@ bool ProjectFileIO::SaveProject(const FilePath &fileName)
    // current to the new file and make it the active file.
    if (mFileName != fileName)
    {
-      auto newDB = CopyTo(fileName);
+      // Do NOT prune here since we need to retain the Undo history
+      // after we switch to the new file.
+      auto newDB = CopyTo(fileName, XO("Saving project"));
       if (!newDB)
       {
-         // LL: I there a problem here??? Looks like RestoreConnection() will
-         //     be run, but no connection has been saved yet.
          return false;
       }
 
@@ -1655,7 +1965,7 @@ bool ProjectFileIO::SaveProject(const FilePath &fileName)
 
 bool ProjectFileIO::SaveCopy(const FilePath& fileName)
 {
-   auto db = CopyTo(fileName, true);
+   auto db = CopyTo(fileName, XO("Backing up project"), true);
    if (!db)
    {
       return false;

src/ProjectFileIO.h

@@ -75,6 +75,7 @@ public:
    bool AutoSave(bool recording = false);
    bool AutoSaveDelete(sqlite3 *db = nullptr);
 
+   bool ImportProject(const FilePath &fileName);
    bool LoadProject(const FilePath &fileName);
    bool SaveProject(const FilePath &fileName);
    bool SaveCopy(const FilePath& fileName);
@@ -112,14 +113,14 @@ private:
 
    void UpdatePrefs() override;
 
-   using ExecCB = std::function<int(wxString *result, int cols, char **vals, char **names)>;
+   using ExecCB = std::function<int(std::vector<wxString> *result, int cols, char **vals, char **names)>;
    struct ExecParm
    {
       ExecCB func;
-      wxString *result;
+      std::vector<wxString> *result;
    };
    static int ExecCallback(void *data, int cols, char **vals, char **names);
-   int Exec(const char *query, ExecCB callback, wxString *result);
+   int Exec(const char *query, ExecCB callback, std::vector<wxString> *result);
 
    // The opening of the database may be delayed until demanded.
    // Returns a non-null pointer to an open database, or throws an exception
@@ -147,6 +148,7 @@ private:
    bool TransactionCommit(const wxString &name);
    bool TransactionRollback(const wxString &name);
 
+   bool GetValues(const char *sql, std::vector<wxString> &value);
    bool GetValue(const char *sql, wxString &value);
    bool GetBlob(const char *sql, wxMemoryBuffer &buffer);
 
@@ -165,7 +167,9 @@ private:
    bool CheckForOrphans(BlockIDs &blockids);
 
    // Return a database connection if successful, which caller must close
-   sqlite3 *CopyTo(const FilePath &destpath, bool prune = false);
+   sqlite3 *CopyTo(const FilePath &destpath,
+                   const TranslatableString &msg,
+                   bool prune = false);
 
    void SetError(const TranslatableString & msg);
    void SetDBError(const TranslatableString & msg);
@@ -173,6 +177,8 @@ private:
    using UpdateCB = std::function<void(int operation, char const *dbname, char const *table, long long rowid)>;
    static void UpdateCallback(void *data, int operation, char const *dbname, char const *table, long long rowid);
 
+   unsigned long long CalculateUsage();
+
 private:
    // non-static data members
    std::weak_ptr<AudacityProject> mpProject;

src/ProjectFileManager.cpp

@@ -831,10 +831,10 @@ void ProjectFileManager::OpenFile(const FilePath &fileNameArg, bool addtohistory
    if (IsAlreadyOpen(fileName))
       return;
 
-   // Data loss may occur if users mistakenly try to open ".aup.bak" files
+   // Data loss may occur if users mistakenly try to open ".aup3.bak" files
    // left over from an unsuccessful save or by previous versions of Audacity.
    // So we always refuse to open such files.
-   if (fileName.Lower().EndsWith(wxT(".aup.bak")))
+   if (fileName.Lower().EndsWith(wxT(".aup3.bak")))
    {
       AudacityMessageBox(
          XO(
@@ -1151,6 +1151,20 @@ bool ProjectFileManager::Import(
    TrackHolders newTracks;
    TranslatableString errorMessage;
 
+   // Handle AUP3 ("project") files directly
+   if (fileName.AfterLast('.').IsSameAs(wxT("aup3"), false)) {
+      auto &projectFileIO = ProjectFileIO::Get(project);
+      if (projectFileIO.ImportProject(fileName)) {
+         auto &history = ProjectHistory::Get(project);
+
+         history.PushState(XO("Imported '%s'").Format(fileName), XO("Import"));
+
+         FileHistory::Global().Append(fileName);
+      }
+
+      return false;
+   }
+
    {
       // Backup Tags, before the import.  Be prepared to roll back changes.
       bool committed = false;
@@ -1193,9 +1207,8 @@ bool ProjectFileManager::Import(
       return false;
    }
 
-   // for AUP ("legacy project") files, do not import the file as if it
+   // Handle AUP ("legacy project") files directly
-   // were an audio file itself
+   if (fileName.AfterLast('.').IsSameAs(wxT("aup3"), false)) {
-   if (fileName.AfterLast('.').IsSameAs(wxT("aup"), false)) {
       auto &history = ProjectHistory::Get( project );
 
       history.PushState(XO("Imported '%s'").Format( fileName ), XO("Import"));

src/ProjectManager.cpp

@@ -674,14 +674,25 @@ void ProjectManager::OnCloseWindow(wxCloseEvent & event)
    // Cleanup the project file
    //
    // Might be that we want to UndoManager::ClearStates() before this???
-   if (!projectFileIO.IsTemporary())
+   bool vacuumed = false;
+   if (!projectFileIO.IsTemporary() && settings.GetCompactAtClose())
    {
-      projectFileIO.Vacuum();
+      vacuumed = projectFileIO.Vacuum();
    }
 
    // See ProjectFileIO::Bypass() for a description
    projectFileIO.Bypass(true);
 
+   // Not sure if this has to come later in the shutdown process, but doing
+   // it before the project window is destroyed looks better when huge states
+   // need to be cleaned up.
+   if (!vacuumed)
+   {
+      // This must be done before the following Deref() since it holds
+      // references to the DirManager.
+      UndoManager::Get( project ).ClearStates();
+   }
+
 #ifdef __WXMAC__
    // Fix bug apparently introduced into 2.1.2 because of wxWidgets 3:
    // closing a project that was made full-screen (as by clicking the green dot
@@ -777,10 +788,6 @@ void ProjectManager::OnCloseWindow(wxCloseEvent & event)
    // Delete all the tracks to free up memory and DirManager references.
    tracks.Clear();
 
-   // This must be done before the following Deref() since it holds
-   // references to the DirManager.
-   UndoManager::Get( project ).ClearStates();
-
    // Remove self from the global array, but defer destruction of self
    auto pSelf = AllProjects{}.Remove( project );
    wxASSERT( pSelf );

src/ProjectSettings.cpp

@@ -91,8 +91,9 @@ ProjectSettings::ProjectSettings(AudacityProject &project)
 
 void ProjectSettings::UpdatePrefs()
 {
+   gPrefs->Read(wxT("/GUI/CompactAtClose"), &mCompactAtClose, true);
    gPrefs->Read(wxT("/AudioFiles/ShowId3Dialog"), &mShowId3Dialog, true);
-   gPrefs->Read(wxT("/GUI/EmptyCanBeDirty"), &mEmptyCanBeDirty, true );
+   gPrefs->Read(wxT("/GUI/EmptyCanBeDirty"), &mEmptyCanBeDirty, true);
    gPrefs->Read(wxT("/GUI/ShowSplashScreen"), &mShowSplashScreen, true);
    mSoloPref = TracksBehaviorsSolo.Read();
    // Update the old default to the NEW default.

src/ProjectSettings.h

@@ -118,6 +118,10 @@ public:
 
    bool GetShowSplashScreen() const { return mShowSplashScreen; }
 
+   // Compact at close
+   void SetCompactAtClose(bool compact) { mCompactAtClose = compact; };
+   bool GetCompactAtClose() const { return mCompactAtClose; }
+
 private:
    void UpdatePrefs() override;
 
@@ -145,6 +149,7 @@ private:
    bool mIsSyncLocked{ false };
    bool mEmptyCanBeDirty;
    bool mShowSplashScreen;
+   bool mCompactAtClose;
 };
 
 #endif

src/import/ImportAUP.cpp

@@ -334,13 +334,17 @@ ProgressResult AUPImportFileHandle::Import(TrackFactory *WXUNUSED(trackFactory),
       return mUpdateResult;
    }
 
+   // Copy the tracks we just created into the project.
    for (auto &track : mTracks)
    {
       tracks.Add(track);
    }
 
+   // Don't need our local track list anymore
    mTracks.clear();
 
+   // If the active project is "dirty", then bypass the below updates as we don't
+   // want to going changing things the user may have already set up.
    if (isDirty)
    {
       return mUpdateResult;
@@ -803,6 +807,7 @@ bool AUPImportFileHandle::HandleProject(XMLTagHandler *&handler)
       {
          set(bandwidthformat, strValue);
       }
+#undef set
    }
 
    if (requiredTags < 3)

src/menus/FileMenus.cpp

@@ -13,6 +13,7 @@
 #include "../ProjectFileManager.h"
 #include "../ProjectHistory.h"
 #include "../ProjectManager.h"
+#include "../ProjectSettings.h"
 #include "../ProjectWindow.h"
 #include "../SelectUtilities.h"
 #include "../TrackPanel.h"
@@ -141,6 +142,21 @@ void OnClose(const CommandContext &context )
    window.Close();
 }
 
+void OnCompact(const CommandContext &context )
+{
+   auto &project = context.project;
+   auto &settings = ProjectSettings::Get( project );
+
+   bool compact;
+   gPrefs->Read(wxT("/GUI/CompactAtClose"), &compact, true);
+
+   compact = !compact;
+   gPrefs->Write(wxT("/GUI/CompactAtClose"), compact);
+   gPrefs->Flush();
+
+   settings.SetCompactAtClose(compact);
+}
+
 void OnSave(const CommandContext &context )
 {
    auto &project = context.project;
@@ -596,7 +612,11 @@ BaseItemSharedPtr FileMenu()
    /////////////////////////////////////////////////////////////////////////////
 
          Command( wxT("Close"), XXO("&Close"), FN(OnClose),
-            AudioIONotBusyFlag(), wxT("Ctrl+W") )
+            AudioIONotBusyFlag(), wxT("Ctrl+W") ),
+
+         Command( wxT("Compact"), XXO("Com&pact at close (on/off)"),
+            FN(OnCompact), AlwaysEnabledFlag,
+            Options{}.CheckTest( wxT("/GUI/CompactAtClose"), true ) )
       ),
 
       Section( "Save",