audacia
/
audacia
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

AUP3: Better space usage calculations 

Several improvements in determining how much actual disk space a
sampleblock uses. This allows us to provide how much space will
be recovered when using File -> Compact Project.

In addition, the History window now provides better space estimates.
This commit is contained in:
Leland Lucius 2020-08-01 04:25:42 -05:00
parent 59c3b360b7
commit 8943494f8a
6 changed files with 282 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake-proxies/sqlite/CMakeLists.txt
View File

@ -20,9 +20,9 @@ list( APPEND INCLUDES
list( APPEND DEFINES
PRIVATE
#
# We need the dbstats table for space calculations.
# We need the dbpage table for space calculations.
#
SQLITE_ENABLE_DBSTAT_VTAB=1
SQLITE_ENABLE_DBPAGE_VTAB=1
# Can't be set after a WAL mode database is initialized, so change
# the default here to ensure all project files get the same page

4
src/DBConnection.h
View File

@ -53,7 +53,9 @@ public:
GetSummary64k,
LoadSampleBlock,
InsertSampleBlock,
DeleteSampleBlock
DeleteSampleBlock,
GetRootPage,
GetDBPage
};
sqlite3_stmt *GetStatement(enum StatementID id);
sqlite3_stmt *Prepare(enum StatementID id, const char *sql);

242
src/ProjectFileIO.cpp
View File

@ -256,11 +256,6 @@ ProjectFileIO::~ProjectFileIO()
{
}
bool ProjectFileIO::OpenProject()
{
return OpenConnection();
}
sqlite3 *ProjectFileIO::DB()
{
auto &curConn = CurrConn();
@ -282,7 +277,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */)
{
auto &curConn = CurrConn();
wxASSERT(!curConn);
bool temp = false;
bool isTemp = false;
if (fileName.empty())
{
@ -290,7 +285,19 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */)
if (fileName.empty())
{
fileName = FileNames::UnsavedProjectFileName();
temp = true;
isTemp = true;
}
}
else
{
// If this project resides in the temporary directory, then we'll mark it
// as temporary.
wxFileName temp(FileNames::TempDir(), wxT(""));
wxFileName file(fileName);
file.SetFullName(wxT(""));
if (file == temp)
{
isTemp = true;
}
}
@ -308,7 +315,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {} */)
return false;
}
mTemporary = temp;
mTemporary = isTemp;
SetFileName(fileName);
@ -944,21 +951,17 @@ bool ProjectFileIO::ShouldCompact(const std::shared_ptr<TrackList> &tracks)
);
// Get the number of blocks and total length from the project file.
unsigned long long total = GetTotalUsage();
unsigned long long blockcount = 0;
unsigned long long total = 0;
auto cb = [&blockcount, &total](int cols, char **vals, char **)
auto cb = [&blockcount](int cols, char **vals, char **)
{
// Convert
wxString(vals[0]).ToULongLong(&blockcount);
wxString(vals[1]).ToULongLong(&total);
return 0;
};
if (!Query("SELECT Count(*), "
"Sum(Length(summary256)) + Sum(Length(summary64k)) + Sum(Length(samples)) "
"FROM sampleblocks;", cb)
|| total == 0)
if (!Query("SELECT Count(*) FROM sampleblocks;", cb) || blockcount == 0)
{
// Shouldn't compact since we don't have the full picture
return false;
@ -1976,6 +1979,11 @@ bool ProjectFileIO::SaveCopy(const FilePath& fileName)
return CopyTo(fileName, XO("Backing up project"), false, true);
}
bool ProjectFileIO::OpenProject()
{
return OpenConnection();
}
bool ProjectFileIO::CloseProject()
{
auto &currConn = CurrConn();
@ -2012,6 +2020,17 @@ bool ProjectFileIO::CloseProject()
return true;
}
bool ProjectFileIO::ReopenProject()
{
FilePath fileName = mFileName;
if (!CloseConnection())
{
return false;
}
return OpenConnection(fileName);
}
bool ProjectFileIO::IsModified() const
{
return mModified;
@ -2051,12 +2070,12 @@ wxLongLong ProjectFileIO::GetFreeDiskSpace()
return -1;
}
const TranslatableString & ProjectFileIO::GetLastError() const
const TranslatableString &ProjectFileIO::GetLastError() const
{
return mLastError;
}
const TranslatableString & ProjectFileIO::GetLibraryError() const
const TranslatableString &ProjectFileIO::GetLibraryError() const
{
return mLibraryError;
}
@ -2118,6 +2137,193 @@ void ProjectFileIO::SetBypass()
return;
}
int64_t ProjectFileIO::GetBlockUsage(SampleBlockID blockid)
{
return GetDiskUsage(CurrConn().get(), blockid);
}
int64_t ProjectFileIO::GetCurrentUsage(const std::shared_ptr<TrackList> &tracks)
{
unsigned long long current = 0;
InspectBlocks(*tracks, BlockSpaceUsageAccumulator(current), nullptr);
return current;
}
int64_t ProjectFileIO::GetTotalUsage()
{
return GetDiskUsage(CurrConn().get(), 0);
}
int64_t ProjectFileIO::GetDiskUsage(DBConnection *conn, SampleBlockID blockid /* = 0 */)
{
typedef struct
{
SampleBlockID pgno;
int currentCell;
int numCells;
unsigned char data[65536];
} page;
std::vector<page> stack;
int64_t total = 0;
int64_t found = 0;
int64_t next = 0;
int rc;
// Get the rootpage for the sampleblocks table.
sqlite3_stmt *stmt = conn->Prepare(DBConnection::GetRootPage,
"SELECT rootpage FROM sqlite_master WHERE tbl_name = 'sampleblocks';");
sqlite3_step(stmt);
int64_t rootpage = sqlite3_column_int64(stmt, 0);
sqlite3_clear_bindings(stmt);
sqlite3_reset(stmt);
// Prepare/retrieve statement to read raw database page
stmt = conn->Prepare(DBConnection::GetDBPage,
"SELECT data FROM sqlite_dbpage WHERE pgno = ?1;");
stack.push_back({rootpage, 0, 0});
do
{
int nd = (stack.size() - 1) * 2;
page &pg = stack.back();
if (pg.numCells == 0)
{
sqlite3_bind_int64(stmt, 1, pg.pgno);
rc = sqlite3_step(stmt);
if (rc != SQLITE_ROW)
{
return found;
}
memcpy(&pg.data,
sqlite3_column_blob(stmt, 0),
sqlite3_column_bytes(stmt, 0));
pg.currentCell = 0;
pg.numCells = get2(&pg.data[3]);
sqlite3_clear_bindings(stmt);
sqlite3_reset(stmt);
}
//wxLogDebug("%*.*spgno %lld currentCell %d numCells %d", nd, nd, "", pg.pgno, pg.currentCell, pg.numCells);
if (pg.data[0] == 0x05)
{
if (pg.currentCell < pg.numCells)
{
next = get4(&pg.data[8]);
bool cont = false;
while (pg.currentCell < pg.numCells)
{
int celloff = get2(&pg.data[12 + (pg.currentCell++ * 2)]);
int pagenum = get4(&pg.data[celloff]);
int64_t intkey = 0;
get_varint(&pg.data[celloff + 4], &intkey);
//wxLogDebug("%*.*sinternal - next %lld celloff %d pagenum %d intkey %lld", nd, nd, " ", next, celloff, pagenum, intkey);
if (!blockid || blockid <= intkey)
{
stack.push_back({pagenum, 0, 0});
cont = true;
break;
}
}
if (cont)
{
continue;
}
}
if (next)
{
stack.push_back({next, 0, 0});
next = 0;
continue;
}
}
else if (pg.data[0] == 0x0d)
{
bool stop = false;
for (int i = 0; i < pg.numCells; i++)
{
int celloff = get2(&pg.data[8 + (i * 2)]);
int64_t payload = 0;
int digits = get_varint(&pg.data[celloff], &payload);
int64_t intkey = 0;
get_varint(&pg.data[celloff + digits], &intkey);
//wxLogDebug("%*.*sleaf - celloff %4d intkey %lld payload %lld", nd, nd, " ", celloff, intkey, payload);
if (blockid)
{
if (blockid == intkey)
{
found = payload;
break;
}
}
else
{
total += payload;
}
}
if (found)
{
break;
}
}
stack.pop_back();
} while (!stack.empty());
return blockid ? found : total;
}
unsigned int ProjectFileIO::get2(const unsigned char *ptr)
{
return (ptr[0] << 8) | ptr[1];
}
unsigned int ProjectFileIO::get4(const unsigned char *ptr)
{
return ((unsigned int) ptr[0] << 24) |
((unsigned int) ptr[1] << 16) |
((unsigned int) ptr[2] << 8) |
((unsigned int) ptr[3]);
}
int ProjectFileIO::get_varint(const unsigned char *ptr, int64_t *out)
{
int64_t val = 0;
int i;
for (i = 0; i < 8; ++i)
{
val = (val << 7) + (ptr[i] & 0x7f);
if ((ptr[i] & 0x80) == 0)
{
*out = val;
return i + 1;
}
}
val = (val << 8) + (ptr[i] & 0xff);
*out = val;
return 9;
}
AutoCommitTransaction::AutoCommitTransaction(ProjectFileIO &projectFileIO,
const char *name)
: mIO(projectFileIO),

24
src/ProjectFileIO.h
View File

@ -38,6 +38,8 @@ using SampleBlockID = long long;
using Connection = std::unique_ptr<DBConnection>;
using BlockIDs = std::unordered_set<SampleBlockID>;
///\brief Object associated with a project that manages reading and writing
/// of Audacity project file formats, and autosave
class ProjectFileIO final
@ -80,6 +82,7 @@ public:
bool OpenProject();
bool CloseProject();
bool ReopenProject();
bool ImportProject(const FilePath &fileName);
bool LoadProject(const FilePath &fileName);
@ -88,6 +91,11 @@ public:
wxLongLong GetFreeDiskSpace();
int64_t GetBlockUsage(SampleBlockID blockid);
int64_t GetCurrentUsage(const std::shared_ptr<TrackList> &tracks);
int64_t GetTotalUsage();
static int64_t GetDiskUsage(DBConnection *conn, SampleBlockID blockid);
const TranslatableString &GetLastError() const;
const TranslatableString &GetLibraryError() const;
@ -118,6 +126,10 @@ public:
bool TransactionCommit(const wxString &name);
bool TransactionRollback(const wxString &name);
// In one SQL command, delete sample blocks with ids in the given set, or
// (when complement is true), with ids not in the given set.
bool DeleteBlocks(const BlockIDs &blockids, bool complement);
// Type of function that is given the fields of one row and returns
// 0 for success or non-zero to stop the query
using ExecCB = std::function<int(int cols, char **vals, char **names)>;
@ -168,15 +180,8 @@ private:
bool WriteDoc(const char *table, const ProjectSerializer &autosave, const char *schema = "main");
// Application defined function to verify blockid exists is in set of blockids
using BlockIDs = std::unordered_set<SampleBlockID>;
static void InSet(sqlite3_context *context, int argc, sqlite3_value **argv);
public:
// In one SQL command, delete sample blocks with ids in the given set, or
// (when complement is true), with ids not in the given set.
bool DeleteBlocks(const BlockIDs &blockids, bool complement);
private:
// Return a database connection if successful, which caller must close
bool CopyTo(const FilePath &destpath,
const TranslatableString &msg,
@ -189,6 +194,11 @@ private:
bool ShouldCompact(const std::shared_ptr<TrackList> &tracks);
// Gets values from SQLite B-tree structures
static unsigned int get2(const unsigned char *ptr);
static unsigned int get4(const unsigned char *ptr);
static int get_varint(const unsigned char *ptr, int64_t *out);
private:
Connection &CurrConn();

24
src/SqliteSampleBlock.cpp
View File

@ -12,6 +12,7 @@ Paul Licameli -- split from SampleBlock.cpp and SampleBlock.h
#include <sqlite3.h>
#include "DBConnection.h"
#include "ProjectFileIO.h"
#include "SampleFormat.h"
#include "xml/XMLTagHandler.h"
@ -454,8 +455,7 @@ MinMaxRMS SqliteSampleBlock::DoGetMinMaxRMS() const
size_t SqliteSampleBlock::GetSpaceUsage() const
{
// Not an exact number, but close enough
return mSummary256Bytes + mSummary64kBytes + mSampleBytes;
return ProjectFileIO::GetDiskUsage(Conn(), mBlockID);
}
size_t SqliteSampleBlock::GetBlob(void *dest,
@ -868,10 +868,16 @@ void SqliteSampleBlock::CalcSummary()
}
// Inject our database implementation at startup
static struct Injector { Injector() {
// Do this some time before the first project is created
(void) SampleBlockFactory::RegisterFactoryFactory(
[]( AudacityProject &project ){
return std::make_shared<SqliteSampleBlockFactory>( project ); }
);
} } injector;
static struct Injector
{
Injector()
{
// Do this some time before the first project is created
(void) SampleBlockFactory::RegisterFactoryFactory(
[]( AudacityProject &project )
{
return std::make_shared<SqliteSampleBlockFactory>( project );
}
);
}
} injector;

33
src/menus/FileMenus.cpp
View File

@ -147,18 +147,35 @@ void OnCompact(const CommandContext &context)
{
auto &project = context.project;
auto &undoManager = UndoManager::Get(project);
auto &clipboard = Clipboard::Get();
auto &projectFileIO = ProjectFileIO::Get(project);
projectFileIO.ReopenProject();
auto currentTracks = TrackList::Create( nullptr );
auto &tracks = TrackList::Get( project );
for (auto t : tracks.Any())
{
currentTracks->Add(t->Duplicate());
}
int64_t total = projectFileIO.GetTotalUsage();
int64_t used = projectFileIO.GetCurrentUsage(currentTracks);
auto before = wxFileName(projectFileIO.GetFileName()).GetSize() +
wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize();
int id = AudacityMessageBox(
XO("Compacting this project will free up disk space by removing unused bytes within the file.\n\n"
"There is %s of free disk space and this project is currently using %s.\n\n"
"NOTE: If you proceed, the current Undo History and clipboard contents will be discarded.\n\n"
"There is %s of free disk space and this project is currently using %s.\n"
"\n"
"If you proceed, the current Undo History and clipboard contents will be discarded "
"and you will recover approximately %s of disk space.\n"
"\n"
"Do you want to continue?")
.Format(Internat::FormatSize(projectFileIO.GetFreeDiskSpace()),
Internat::FormatSize(before.GetValue())),
Internat::FormatSize(before.GetValue()),
Internat::FormatSize(total - used)),
XO("Compact Project"),
wxYES_NO);
@ -173,23 +190,15 @@ void OnCompact(const CommandContext &context)
auto numStates = undoManager.GetNumStates();
undoManager.RemoveStates(numStates - 1);
auto &clipboard = Clipboard::Get();
clipboard.Clear();
auto currentTracks = TrackList::Create( nullptr );
auto &tracks = TrackList::Get( project );
for (auto t : tracks.Any())
{
currentTracks->Add(t->Duplicate());
}
projectFileIO.Compact(currentTracks, true);
auto after = wxFileName(projectFileIO.GetFileName()).GetSize() +
wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize();
AudacityMessageBox(
XO("Compacting freed %s of disk space.")
XO("Compacting actually freed %s of disk space.")
.Format(Internat::FormatSize((before - after).GetValue())),
XO("Compact Project"));
}