From 8943494f8a394c34538ac8549ee59ad1da55b0e8 Mon Sep 17 00:00:00 2001 From: Leland Lucius Date: Sat, 1 Aug 2020 04:25:42 -0500 Subject: [PATCH] AUP3: Better space usage calculations Several improvements in determining how much actual disk space a sampleblock uses. This allows us to provide how much space will be recovered when using File -> Compact Project. In addition, the History window now provides better space estimates. --- cmake-proxies/sqlite/CMakeLists.txt | 4 +- src/DBConnection.h | 4 +- src/ProjectFileIO.cpp | 242 +++++++++++++++++++++++++--- src/ProjectFileIO.h | 24 ++- src/SqliteSampleBlock.cpp | 24 +-- src/menus/FileMenus.cpp | 33 ++-- 6 files changed, 282 insertions(+), 49 deletions(-) diff --git a/cmake-proxies/sqlite/CMakeLists.txt b/cmake-proxies/sqlite/CMakeLists.txt index 0de801f3f..63d70637c 100644 --- a/cmake-proxies/sqlite/CMakeLists.txt +++ b/cmake-proxies/sqlite/CMakeLists.txt @@ -20,9 +20,9 @@ list( APPEND INCLUDES list( APPEND DEFINES PRIVATE # - # We need the dbstats table for space calculations. + # We need the dbpage table for space calculations. # - SQLITE_ENABLE_DBSTAT_VTAB=1 + SQLITE_ENABLE_DBPAGE_VTAB=1 # Can't be set after a WAL mode database is initialized, so change # the default here to ensure all project files get the same page diff --git a/src/DBConnection.h b/src/DBConnection.h index 4c4714b82..d2dd7f5b6 100644 --- a/src/DBConnection.h +++ b/src/DBConnection.h @@ -53,7 +53,9 @@ public: GetSummary64k, LoadSampleBlock, InsertSampleBlock, - DeleteSampleBlock + DeleteSampleBlock, + GetRootPage, + GetDBPage }; sqlite3_stmt *GetStatement(enum StatementID id); sqlite3_stmt *Prepare(enum StatementID id, const char *sql); diff --git a/src/ProjectFileIO.cpp b/src/ProjectFileIO.cpp index 9f2aab9db..df8a5b21c 100644 --- a/src/ProjectFileIO.cpp +++ b/src/ProjectFileIO.cpp @@ -256,11 +256,6 @@ ProjectFileIO::~ProjectFileIO() { } -bool ProjectFileIO::OpenProject() -{ - return OpenConnection(); -} - sqlite3 *ProjectFileIO::DB() { auto &curConn = CurrConn(); @@ -282,7 +277,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */) { auto &curConn = CurrConn(); wxASSERT(!curConn); - bool temp = false; + bool isTemp = false; if (fileName.empty()) { @@ -290,7 +285,19 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */) if (fileName.empty()) { fileName = FileNames::UnsavedProjectFileName(); - temp = true; + isTemp = true; + } + } + else + { + // If this project resides in the temporary directory, then we'll mark it + // as temporary. + wxFileName temp(FileNames::TempDir(), wxT("")); + wxFileName file(fileName); + file.SetFullName(wxT("")); + if (file == temp) + { + isTemp = true; } } @@ -308,7 +315,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */) return false; } - mTemporary = temp; + mTemporary = isTemp; SetFileName(fileName); @@ -944,21 +951,17 @@ bool ProjectFileIO::ShouldCompact(const std::shared_ptr &tracks) ); // Get the number of blocks and total length from the project file. + unsigned long long total = GetTotalUsage(); unsigned long long blockcount = 0; - unsigned long long total = 0; - - auto cb = [&blockcount, &total](int cols, char **vals, char **) + + auto cb = [&blockcount](int cols, char **vals, char **) { // Convert wxString(vals[0]).ToULongLong(&blockcount); - wxString(vals[1]).ToULongLong(&total); return 0; }; - if (!Query("SELECT Count(*), " - "Sum(Length(summary256)) + Sum(Length(summary64k)) + Sum(Length(samples)) " - "FROM sampleblocks;", cb) - || total == 0) + if (!Query("SELECT Count(*) FROM sampleblocks;", cb) || blockcount == 0) { // Shouldn't compact since we don't have the full picture return false; @@ -1976,6 +1979,11 @@ bool ProjectFileIO::SaveCopy(const FilePath& fileName) return CopyTo(fileName, XO("Backing up project"), false, true); } +bool ProjectFileIO::OpenProject() +{ + return OpenConnection(); +} + bool ProjectFileIO::CloseProject() { auto &currConn = CurrConn(); @@ -2012,6 +2020,17 @@ bool ProjectFileIO::CloseProject() return true; } +bool ProjectFileIO::ReopenProject() +{ + FilePath fileName = mFileName; + if (!CloseConnection()) + { + return false; + } + + return OpenConnection(fileName); +} + bool ProjectFileIO::IsModified() const { return mModified; @@ -2051,12 +2070,12 @@ wxLongLong ProjectFileIO::GetFreeDiskSpace() return -1; } -const TranslatableString & ProjectFileIO::GetLastError() const +const TranslatableString &ProjectFileIO::GetLastError() const { return mLastError; } -const TranslatableString & ProjectFileIO::GetLibraryError() const +const TranslatableString &ProjectFileIO::GetLibraryError() const { return mLibraryError; } @@ -2118,6 +2137,193 @@ void ProjectFileIO::SetBypass() return; } +int64_t ProjectFileIO::GetBlockUsage(SampleBlockID blockid) +{ + return GetDiskUsage(CurrConn().get(), blockid); +} + +int64_t ProjectFileIO::GetCurrentUsage(const std::shared_ptr &tracks) +{ + unsigned long long current = 0; + + InspectBlocks(*tracks, BlockSpaceUsageAccumulator(current), nullptr); + + return current; +} + +int64_t ProjectFileIO::GetTotalUsage() +{ + return GetDiskUsage(CurrConn().get(), 0); +} + +int64_t ProjectFileIO::GetDiskUsage(DBConnection *conn, SampleBlockID blockid /* = 0 */) +{ + typedef struct + { + SampleBlockID pgno; + int currentCell; + int numCells; + unsigned char data[65536]; + } page; + std::vector stack; + + int64_t total = 0; + int64_t found = 0; + int64_t next = 0; + int rc; + + // Get the rootpage for the sampleblocks table. + sqlite3_stmt *stmt = conn->Prepare(DBConnection::GetRootPage, + "SELECT rootpage FROM sqlite_master WHERE tbl_name = 'sampleblocks';"); + sqlite3_step(stmt); + int64_t rootpage = sqlite3_column_int64(stmt, 0); + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + + // Prepare/retrieve statement to read raw database page + stmt = conn->Prepare(DBConnection::GetDBPage, + "SELECT data FROM sqlite_dbpage WHERE pgno = ?1;"); + + stack.push_back({rootpage, 0, 0}); + do + { + int nd = (stack.size() - 1) * 2; + page &pg = stack.back(); + + if (pg.numCells == 0) + { + sqlite3_bind_int64(stmt, 1, pg.pgno); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_ROW) + { + return found; + } + + memcpy(&pg.data, + sqlite3_column_blob(stmt, 0), + sqlite3_column_bytes(stmt, 0)); + + pg.currentCell = 0; + pg.numCells = get2(&pg.data[3]); + + sqlite3_clear_bindings(stmt); + sqlite3_reset(stmt); + } + + //wxLogDebug("%*.*spgno %lld currentCell %d numCells %d", nd, nd, "", pg.pgno, pg.currentCell, pg.numCells); + if (pg.data[0] == 0x05) + { + if (pg.currentCell < pg.numCells) + { + next = get4(&pg.data[8]); + + bool cont = false; + while (pg.currentCell < pg.numCells) + { + int celloff = get2(&pg.data[12 + (pg.currentCell++ * 2)]); + + int pagenum = get4(&pg.data[celloff]); + + int64_t intkey = 0; + get_varint(&pg.data[celloff + 4], &intkey); + + //wxLogDebug("%*.*sinternal - next %lld celloff %d pagenum %d intkey %lld", nd, nd, " ", next, celloff, pagenum, intkey); + if (!blockid || blockid <= intkey) + { + stack.push_back({pagenum, 0, 0}); + cont = true; + break; + } + } + + if (cont) + { + continue; + } + } + + if (next) + { + stack.push_back({next, 0, 0}); + next = 0; + continue; + } + + } + else if (pg.data[0] == 0x0d) + { + bool stop = false; + for (int i = 0; i < pg.numCells; i++) + { + int celloff = get2(&pg.data[8 + (i * 2)]); + + int64_t payload = 0; + int digits = get_varint(&pg.data[celloff], &payload); + + int64_t intkey = 0; + get_varint(&pg.data[celloff + digits], &intkey); + //wxLogDebug("%*.*sleaf - celloff %4d intkey %lld payload %lld", nd, nd, " ", celloff, intkey, payload); + + if (blockid) + { + if (blockid == intkey) + { + found = payload; + break; + } + } + else + { + total += payload; + } + } + + if (found) + { + break; + } + } + + stack.pop_back(); + } while (!stack.empty()); + + return blockid ? found : total; +} + +unsigned int ProjectFileIO::get2(const unsigned char *ptr) +{ + return (ptr[0] << 8) | ptr[1]; +} + +unsigned int ProjectFileIO::get4(const unsigned char *ptr) +{ + return ((unsigned int) ptr[0] << 24) | + ((unsigned int) ptr[1] << 16) | + ((unsigned int) ptr[2] << 8) | + ((unsigned int) ptr[3]); +} + +int ProjectFileIO::get_varint(const unsigned char *ptr, int64_t *out) +{ + int64_t val = 0; + int i; + for (i = 0; i < 8; ++i) + { + val = (val << 7) + (ptr[i] & 0x7f); + if ((ptr[i] & 0x80) == 0) + { + *out = val; + return i + 1; + } + } + + val = (val << 8) + (ptr[i] & 0xff); + *out = val; + + return 9; +} + AutoCommitTransaction::AutoCommitTransaction(ProjectFileIO &projectFileIO, const char *name) : mIO(projectFileIO), diff --git a/src/ProjectFileIO.h b/src/ProjectFileIO.h index 34a6298dc..f474e6448 100644 --- a/src/ProjectFileIO.h +++ b/src/ProjectFileIO.h @@ -38,6 +38,8 @@ using SampleBlockID = long long; using Connection = std::unique_ptr; +using BlockIDs = std::unordered_set; + ///\brief Object associated with a project that manages reading and writing /// of Audacity project file formats, and autosave class ProjectFileIO final @@ -80,6 +82,7 @@ public: bool OpenProject(); bool CloseProject(); + bool ReopenProject(); bool ImportProject(const FilePath &fileName); bool LoadProject(const FilePath &fileName); @@ -88,6 +91,11 @@ public: wxLongLong GetFreeDiskSpace(); + int64_t GetBlockUsage(SampleBlockID blockid); + int64_t GetCurrentUsage(const std::shared_ptr &tracks); + int64_t GetTotalUsage(); + static int64_t GetDiskUsage(DBConnection *conn, SampleBlockID blockid); + const TranslatableString &GetLastError() const; const TranslatableString &GetLibraryError() const; @@ -118,6 +126,10 @@ public: bool TransactionCommit(const wxString &name); bool TransactionRollback(const wxString &name); + // In one SQL command, delete sample blocks with ids in the given set, or + // (when complement is true), with ids not in the given set. + bool DeleteBlocks(const BlockIDs &blockids, bool complement); + // Type of function that is given the fields of one row and returns // 0 for success or non-zero to stop the query using ExecCB = std::function; @@ -168,15 +180,8 @@ private: bool WriteDoc(const char *table, const ProjectSerializer &autosave, const char *schema = "main"); // Application defined function to verify blockid exists is in set of blockids - using BlockIDs = std::unordered_set; static void InSet(sqlite3_context *context, int argc, sqlite3_value **argv); -public: - // In one SQL command, delete sample blocks with ids in the given set, or - // (when complement is true), with ids not in the given set. - bool DeleteBlocks(const BlockIDs &blockids, bool complement); - -private: // Return a database connection if successful, which caller must close bool CopyTo(const FilePath &destpath, const TranslatableString &msg, @@ -189,6 +194,11 @@ private: bool ShouldCompact(const std::shared_ptr &tracks); + // Gets values from SQLite B-tree structures + static unsigned int get2(const unsigned char *ptr); + static unsigned int get4(const unsigned char *ptr); + static int get_varint(const unsigned char *ptr, int64_t *out); + private: Connection &CurrConn(); diff --git a/src/SqliteSampleBlock.cpp b/src/SqliteSampleBlock.cpp index 19bb4164b..589d7df51 100644 --- a/src/SqliteSampleBlock.cpp +++ b/src/SqliteSampleBlock.cpp @@ -12,6 +12,7 @@ Paul Licameli -- split from SampleBlock.cpp and SampleBlock.h #include #include "DBConnection.h" +#include "ProjectFileIO.h" #include "SampleFormat.h" #include "xml/XMLTagHandler.h" @@ -454,8 +455,7 @@ MinMaxRMS SqliteSampleBlock::DoGetMinMaxRMS() const size_t SqliteSampleBlock::GetSpaceUsage() const { - // Not an exact number, but close enough - return mSummary256Bytes + mSummary64kBytes + mSampleBytes; + return ProjectFileIO::GetDiskUsage(Conn(), mBlockID); } size_t SqliteSampleBlock::GetBlob(void *dest, @@ -868,10 +868,16 @@ void SqliteSampleBlock::CalcSummary() } // Inject our database implementation at startup -static struct Injector { Injector() { - // Do this some time before the first project is created - (void) SampleBlockFactory::RegisterFactoryFactory( - []( AudacityProject &project ){ - return std::make_shared( project ); } - ); -} } injector; +static struct Injector +{ + Injector() + { + // Do this some time before the first project is created + (void) SampleBlockFactory::RegisterFactoryFactory( + []( AudacityProject &project ) + { + return std::make_shared( project ); + } + ); + } +} injector; diff --git a/src/menus/FileMenus.cpp b/src/menus/FileMenus.cpp index 7e44e34d7..4ab6ee408 100644 --- a/src/menus/FileMenus.cpp +++ b/src/menus/FileMenus.cpp @@ -147,18 +147,35 @@ void OnCompact(const CommandContext &context) { auto &project = context.project; auto &undoManager = UndoManager::Get(project); + auto &clipboard = Clipboard::Get(); auto &projectFileIO = ProjectFileIO::Get(project); + projectFileIO.ReopenProject(); + + auto currentTracks = TrackList::Create( nullptr ); + auto &tracks = TrackList::Get( project ); + for (auto t : tracks.Any()) + { + currentTracks->Add(t->Duplicate()); + } + + int64_t total = projectFileIO.GetTotalUsage(); + int64_t used = projectFileIO.GetCurrentUsage(currentTracks); + auto before = wxFileName(projectFileIO.GetFileName()).GetSize() + wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize(); int id = AudacityMessageBox( XO("Compacting this project will free up disk space by removing unused bytes within the file.\n\n" - "There is %s of free disk space and this project is currently using %s.\n\n" - "NOTE: If you proceed, the current Undo History and clipboard contents will be discarded.\n\n" + "There is %s of free disk space and this project is currently using %s.\n" + "\n" + "If you proceed, the current Undo History and clipboard contents will be discarded " + "and you will recover approximately %s of disk space.\n" + "\n" "Do you want to continue?") .Format(Internat::FormatSize(projectFileIO.GetFreeDiskSpace()), - Internat::FormatSize(before.GetValue())), + Internat::FormatSize(before.GetValue()), + Internat::FormatSize(total - used)), XO("Compact Project"), wxYES_NO); @@ -173,23 +190,15 @@ void OnCompact(const CommandContext &context) auto numStates = undoManager.GetNumStates(); undoManager.RemoveStates(numStates - 1); - auto &clipboard = Clipboard::Get(); clipboard.Clear(); - auto currentTracks = TrackList::Create( nullptr ); - auto &tracks = TrackList::Get( project ); - for (auto t : tracks.Any()) - { - currentTracks->Add(t->Duplicate()); - } - projectFileIO.Compact(currentTracks, true); auto after = wxFileName(projectFileIO.GetFileName()).GetSize() + wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize(); AudacityMessageBox( - XO("Compacting freed %s of disk space.") + XO("Compacting actually freed %s of disk space.") .Format(Internat::FormatSize((before - after).GetValue())), XO("Compact Project")); }