AUP3: Better space usage calculations

Several improvements in determining how much actual disk space a
sampleblock uses. This allows us to provide how much space will
be recovered when using File -> Compact Project.

In addition, the History window now provides better space estimates.

cmake-proxies/sqlite/CMakeLists.txt

@@ -20,9 +20,9 @@ list( APPEND INCLUDES
 list( APPEND DEFINES
    PRIVATE
       #
-      # We need the dbstats table for space calculations.
+      # We need the dbpage table for space calculations.
       #
-      SQLITE_ENABLE_DBSTAT_VTAB=1
+      SQLITE_ENABLE_DBPAGE_VTAB=1
 
       # Can't be set after a WAL mode database is initialized, so change
       # the default here to ensure all project files get the same page 

src/DBConnection.h

@@ -53,7 +53,9 @@ public:
       GetSummary64k,
       LoadSampleBlock,
       InsertSampleBlock,
-      DeleteSampleBlock
+      DeleteSampleBlock,
+      GetRootPage,
+      GetDBPage
    };
    sqlite3_stmt *GetStatement(enum StatementID id);
    sqlite3_stmt *Prepare(enum StatementID id, const char *sql);

src/ProjectFileIO.cpp

@@ -256,11 +256,6 @@ ProjectFileIO::~ProjectFileIO()
 {
 }
 
-bool ProjectFileIO::OpenProject()
-{
-   return OpenConnection();
-}
-
 sqlite3 *ProjectFileIO::DB()
 {
    auto &curConn = CurrConn();
@@ -282,7 +277,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {}  */)
 {
    auto &curConn = CurrConn();
    wxASSERT(!curConn);
-   bool temp = false;
+   bool isTemp = false;
 
    if (fileName.empty())
    {
@@ -290,7 +285,19 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {}  */)
       if (fileName.empty())
       {
          fileName = FileNames::UnsavedProjectFileName();
-         temp = true;
+         isTemp = true;
+      }
+   }
+   else
+   {
+      // If this project resides in the temporary directory, then we'll mark it
+      // as temporary.
+      wxFileName temp(FileNames::TempDir(), wxT(""));
+      wxFileName file(fileName);
+      file.SetFullName(wxT(""));
+      if (file == temp)
+      {
+         isTemp = true;
       }
    }
 
@@ -308,7 +315,7 @@ bool ProjectFileIO::OpenConnection(FilePath fileName /* = {}  */)
       return false;
    }
 
-   mTemporary = temp;
+   mTemporary = isTemp;
 
    SetFileName(fileName);
 
@@ -944,21 +951,17 @@ bool ProjectFileIO::ShouldCompact(const std::shared_ptr<TrackList> &tracks)
    );
 
    // Get the number of blocks and total length from the project file.
+   unsigned long long total = GetTotalUsage();
    unsigned long long blockcount = 0;
-   unsigned long long total = 0;
+   
-
+   auto cb = [&blockcount](int cols, char **vals, char **)
-   auto cb = [&blockcount, &total](int cols, char **vals, char **)
    {
       // Convert
       wxString(vals[0]).ToULongLong(&blockcount);
-      wxString(vals[1]).ToULongLong(&total);
       return 0;
    };
 
-   if (!Query("SELECT Count(*), "
+   if (!Query("SELECT Count(*) FROM sampleblocks;", cb) || blockcount == 0)
-     "Sum(Length(summary256)) + Sum(Length(summary64k)) + Sum(Length(samples)) "
-     "FROM sampleblocks;", cb)
-       || total == 0)
    {
       // Shouldn't compact since we don't have the full picture
       return false;
@@ -1976,6 +1979,11 @@ bool ProjectFileIO::SaveCopy(const FilePath& fileName)
    return CopyTo(fileName, XO("Backing up project"), false, true);
 }
 
+bool ProjectFileIO::OpenProject()
+{
+   return OpenConnection();
+}
+
 bool ProjectFileIO::CloseProject()
 {
    auto &currConn = CurrConn();
@@ -2012,6 +2020,17 @@ bool ProjectFileIO::CloseProject()
    return true;
 }
 
+bool ProjectFileIO::ReopenProject()
+{
+   FilePath fileName = mFileName;
+   if (!CloseConnection())
+   {
+      return false;
+   }
+
+   return OpenConnection(fileName);
+}
+
 bool ProjectFileIO::IsModified() const
 {
    return mModified;
@@ -2051,12 +2070,12 @@ wxLongLong ProjectFileIO::GetFreeDiskSpace()
    return -1;
 }
 
-const TranslatableString & ProjectFileIO::GetLastError() const
+const TranslatableString &ProjectFileIO::GetLastError() const
 {
    return mLastError;
 }
 
-const TranslatableString & ProjectFileIO::GetLibraryError() const
+const TranslatableString &ProjectFileIO::GetLibraryError() const
 {
    return mLibraryError;
 }
@@ -2118,6 +2137,193 @@ void ProjectFileIO::SetBypass()
    return;
 }
 
+int64_t ProjectFileIO::GetBlockUsage(SampleBlockID blockid)
+{
+   return GetDiskUsage(CurrConn().get(), blockid);
+}
+
+int64_t ProjectFileIO::GetCurrentUsage(const std::shared_ptr<TrackList> &tracks)
+{
+   unsigned long long current = 0;
+
+   InspectBlocks(*tracks, BlockSpaceUsageAccumulator(current), nullptr);
+
+   return current;
+}
+
+int64_t ProjectFileIO::GetTotalUsage()
+{
+   return GetDiskUsage(CurrConn().get(), 0);
+}
+
+int64_t ProjectFileIO::GetDiskUsage(DBConnection *conn, SampleBlockID blockid /* = 0 */)
+{
+   typedef struct
+   {
+      SampleBlockID pgno;
+      int currentCell;
+      int numCells;
+      unsigned char data[65536];
+   } page;
+   std::vector<page> stack;
+
+   int64_t total = 0;
+   int64_t found = 0;
+   int64_t next = 0;
+   int rc;
+
+   // Get the rootpage for the sampleblocks table.
+   sqlite3_stmt *stmt = conn->Prepare(DBConnection::GetRootPage,
+      "SELECT rootpage FROM sqlite_master WHERE tbl_name = 'sampleblocks';");
+   sqlite3_step(stmt);
+   int64_t rootpage = sqlite3_column_int64(stmt, 0);
+   sqlite3_clear_bindings(stmt);
+   sqlite3_reset(stmt);
+
+   // Prepare/retrieve statement to read raw database page
+   stmt = conn->Prepare(DBConnection::GetDBPage,
+      "SELECT data FROM sqlite_dbpage WHERE pgno = ?1;");
+
+   stack.push_back({rootpage, 0, 0});
+   do
+   {
+      int nd = (stack.size() - 1) * 2;
+      page &pg = stack.back();
+
+      if (pg.numCells == 0)
+      {
+         sqlite3_bind_int64(stmt, 1, pg.pgno);
+
+         rc = sqlite3_step(stmt);
+         if (rc != SQLITE_ROW)
+         {
+            return found;
+         }
+
+         memcpy(&pg.data,
+                  sqlite3_column_blob(stmt, 0),
+                  sqlite3_column_bytes(stmt, 0));
+
+         pg.currentCell = 0;
+         pg.numCells = get2(&pg.data[3]);
+
+         sqlite3_clear_bindings(stmt);
+         sqlite3_reset(stmt);
+      }
+
+      //wxLogDebug("%*.*spgno %lld currentCell %d numCells %d", nd, nd, "", pg.pgno, pg.currentCell, pg.numCells);
+      if (pg.data[0] == 0x05)
+      {
+         if (pg.currentCell < pg.numCells)
+         {
+            next = get4(&pg.data[8]);
+
+            bool cont = false;
+            while (pg.currentCell < pg.numCells)
+            {
+               int celloff = get2(&pg.data[12 + (pg.currentCell++ * 2)]);
+
+               int pagenum = get4(&pg.data[celloff]);
+
+               int64_t intkey = 0;
+               get_varint(&pg.data[celloff + 4], &intkey);
+
+               //wxLogDebug("%*.*sinternal - next %lld celloff %d pagenum %d intkey %lld", nd, nd, " ", next, celloff, pagenum, intkey);
+               if (!blockid || blockid <= intkey)
+               {
+                  stack.push_back({pagenum, 0, 0});
+                  cont = true;
+                  break;
+               }
+            }
+
+            if (cont)
+            {
+               continue;
+            }
+         }
+
+         if (next)
+         {
+            stack.push_back({next, 0, 0});
+            next = 0;
+            continue;
+         }
+
+      }
+      else if (pg.data[0] == 0x0d)
+      {
+         bool stop = false;
+         for (int i = 0; i < pg.numCells; i++)
+         {
+            int celloff = get2(&pg.data[8 + (i * 2)]);
+
+            int64_t payload = 0;
+            int digits = get_varint(&pg.data[celloff], &payload);
+
+            int64_t intkey = 0;
+            get_varint(&pg.data[celloff + digits], &intkey);
+            //wxLogDebug("%*.*sleaf - celloff %4d intkey %lld payload %lld", nd, nd, " ", celloff, intkey, payload);
+
+            if (blockid)
+            {
+               if (blockid == intkey)
+               {
+                  found = payload;
+                  break;
+               }
+            }
+            else
+            {
+               total += payload;
+            }
+         }
+
+         if (found)
+         {
+            break;
+         }
+      }
+
+      stack.pop_back();
+   } while (!stack.empty());
+
+   return blockid ? found : total;
+}
+
+unsigned int ProjectFileIO::get2(const unsigned char *ptr)
+{
+   return (ptr[0] << 8) | ptr[1];
+}
+
+unsigned int ProjectFileIO::get4(const unsigned char *ptr)
+{
+   return ((unsigned int) ptr[0] << 24) |
+          ((unsigned int) ptr[1] << 16) |
+          ((unsigned int) ptr[2] << 8)  |
+          ((unsigned int) ptr[3]);
+}
+
+int ProjectFileIO::get_varint(const unsigned char *ptr, int64_t *out)
+{
+   int64_t val = 0;
+   int i;
+   for (i = 0; i < 8; ++i)
+   {
+      val = (val << 7) + (ptr[i] & 0x7f);
+      if ((ptr[i] & 0x80) == 0)
+      {
+         *out = val;
+         return i + 1;
+      }
+   }
+
+   val = (val << 8) + (ptr[i] & 0xff);
+   *out = val;
+
+   return 9;
+}
+
 AutoCommitTransaction::AutoCommitTransaction(ProjectFileIO &projectFileIO,
                                              const char *name)
 :  mIO(projectFileIO),

src/ProjectFileIO.h

@@ -38,6 +38,8 @@ using SampleBlockID = long long;
 
 using Connection = std::unique_ptr<DBConnection>;
 
+using BlockIDs = std::unordered_set<SampleBlockID>;
+
 ///\brief Object associated with a project that manages reading and writing
 /// of Audacity project file formats, and autosave
 class ProjectFileIO final
@@ -80,6 +82,7 @@ public:
 
    bool OpenProject();
    bool CloseProject();
+   bool ReopenProject();
 
    bool ImportProject(const FilePath &fileName);
    bool LoadProject(const FilePath &fileName);
@@ -88,6 +91,11 @@ public:
 
    wxLongLong GetFreeDiskSpace();
 
+   int64_t GetBlockUsage(SampleBlockID blockid);
+   int64_t GetCurrentUsage(const std::shared_ptr<TrackList> &tracks);
+   int64_t GetTotalUsage();
+   static int64_t GetDiskUsage(DBConnection *conn, SampleBlockID blockid);
+
    const TranslatableString &GetLastError() const;
    const TranslatableString &GetLibraryError() const;
 
@@ -118,6 +126,10 @@ public:
    bool TransactionCommit(const wxString &name);
    bool TransactionRollback(const wxString &name);
 
+   // In one SQL command, delete sample blocks with ids in the given set, or
+   // (when complement is true), with ids not in the given set.
+   bool DeleteBlocks(const BlockIDs &blockids, bool complement);
+
    // Type of function that is given the fields of one row and returns
    // 0 for success or non-zero to stop the query
    using ExecCB = std::function<int(int cols, char **vals, char **names)>;
@@ -168,15 +180,8 @@ private:
    bool WriteDoc(const char *table, const ProjectSerializer &autosave, const char *schema = "main");
 
    // Application defined function to verify blockid exists is in set of blockids
-   using BlockIDs = std::unordered_set<SampleBlockID>;
    static void InSet(sqlite3_context *context, int argc, sqlite3_value **argv);
 
-public:
-   // In one SQL command, delete sample blocks with ids in the given set, or
-   // (when complement is true), with ids not in the given set.
-   bool DeleteBlocks(const BlockIDs &blockids, bool complement);
-
-private:
    // Return a database connection if successful, which caller must close
    bool CopyTo(const FilePath &destpath,
                const TranslatableString &msg,
@@ -189,6 +194,11 @@ private:
 
    bool ShouldCompact(const std::shared_ptr<TrackList> &tracks);
 
+   // Gets values from SQLite B-tree structures
+   static unsigned int get2(const unsigned char *ptr);
+   static unsigned int get4(const unsigned char *ptr);
+   static int get_varint(const unsigned char *ptr, int64_t *out);
+
 private:
    Connection &CurrConn();
 

src/SqliteSampleBlock.cpp

@@ -12,6 +12,7 @@ Paul Licameli -- split from SampleBlock.cpp and SampleBlock.h
 #include <sqlite3.h>
 
 #include "DBConnection.h"
+#include "ProjectFileIO.h"
 #include "SampleFormat.h"
 #include "xml/XMLTagHandler.h"
 
@@ -454,8 +455,7 @@ MinMaxRMS SqliteSampleBlock::DoGetMinMaxRMS() const
 
 size_t SqliteSampleBlock::GetSpaceUsage() const
 {
-   // Not an exact number, but close enough
+   return ProjectFileIO::GetDiskUsage(Conn(), mBlockID);
-   return mSummary256Bytes + mSummary64kBytes + mSampleBytes;
 }
 
 size_t SqliteSampleBlock::GetBlob(void *dest,
@@ -868,10 +868,16 @@ void SqliteSampleBlock::CalcSummary()
 }
 
 // Inject our database implementation at startup
-static struct Injector { Injector() {
+static struct Injector
-   // Do this some time before the first project is created
+{
-   (void) SampleBlockFactory::RegisterFactoryFactory(
+   Injector()
-      []( AudacityProject &project ){
+   {
-         return std::make_shared<SqliteSampleBlockFactory>( project ); }
+      // Do this some time before the first project is created
-   );
+      (void) SampleBlockFactory::RegisterFactoryFactory(
-} } injector;
+         []( AudacityProject &project )
+         {
+            return std::make_shared<SqliteSampleBlockFactory>( project );
+         }
+      );
+   }
+} injector;

src/menus/FileMenus.cpp

@@ -147,18 +147,35 @@ void OnCompact(const CommandContext &context)
 {
    auto &project = context.project;
    auto &undoManager = UndoManager::Get(project);
+   auto &clipboard = Clipboard::Get();
    auto &projectFileIO = ProjectFileIO::Get(project);
 
+   projectFileIO.ReopenProject();
+
+   auto currentTracks = TrackList::Create( nullptr );
+   auto &tracks = TrackList::Get( project );
+   for (auto t : tracks.Any())
+   {
+      currentTracks->Add(t->Duplicate());
+   }
+
+   int64_t total = projectFileIO.GetTotalUsage();
+   int64_t used = projectFileIO.GetCurrentUsage(currentTracks);
+
    auto before = wxFileName(projectFileIO.GetFileName()).GetSize() +
                  wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize();
 
    int id = AudacityMessageBox(
       XO("Compacting this project will free up disk space by removing unused bytes within the file.\n\n"
-         "There is %s of free disk space and this project is currently using %s.\n\n"
+         "There is %s of free disk space and this project is currently using %s.\n"
-         "NOTE: If you proceed, the current Undo History and clipboard contents will be discarded.\n\n"
+         "\n"
+         "If you proceed, the current Undo History and clipboard contents will be discarded "
+         "and you will recover approximately %s of disk space.\n"
+         "\n"
          "Do you want to continue?")
       .Format(Internat::FormatSize(projectFileIO.GetFreeDiskSpace()),
-              Internat::FormatSize(before.GetValue())),
+              Internat::FormatSize(before.GetValue()),
+              Internat::FormatSize(total - used)),
       XO("Compact Project"),
       wxYES_NO);
 
@@ -173,23 +190,15 @@ void OnCompact(const CommandContext &context)
    auto numStates = undoManager.GetNumStates();
    undoManager.RemoveStates(numStates - 1);
 
-   auto &clipboard = Clipboard::Get();
    clipboard.Clear();
 
-   auto currentTracks = TrackList::Create( nullptr );
-   auto &tracks = TrackList::Get( project );
-   for (auto t : tracks.Any())
-   {
-      currentTracks->Add(t->Duplicate());
-   }
-
    projectFileIO.Compact(currentTracks, true);
 
    auto after = wxFileName(projectFileIO.GetFileName()).GetSize() +
                 wxFileName(projectFileIO.GetFileName() + wxT("-wal")).GetSize();
 
    AudacityMessageBox(
-      XO("Compacting freed %s of disk space.")
+      XO("Compacting actually freed %s of disk space.")
       .Format(Internat::FormatSize((before - after).GetValue())),
       XO("Compact Project"));
 }