AUP3: Reworked CopyTo() yet again

This time it has the potential to produce much smaller
output files since it ONLY copies the active blocks and
not all of the blocks related to undo history.

src/ProjectFileIO.cpp

@@ -19,8 +19,10 @@ Paul Licameli split from AudacityProject.cpp
 #include "ProjectFileIORegistry.h"
 #include "ProjectSerializer.h"
 #include "ProjectSettings.h"
+#include "SampleBlock.h"
 #include "Tags.h"
 #include "ViewInfo.h"
+#include "WaveClip.h"
 #include "WaveTrack.h"
 #include "widgets/AudacityMessageBox.h"
 #include "widgets/NumericTextCtrl.h"
@@ -41,10 +43,10 @@ static const int ProjectFileVersion = 1;
 // to sampleblocks.
 
 static const char *ProjectFileSchema =
-   "PRAGMA application_id = %d;"
+   "PRAGMA <dbname>.application_id = %d;"
-   "PRAGMA user_version = %d;"
+   "PRAGMA <dbname>.user_version = %d;"
-   "PRAGMA journal_mode = WAL;"
+   "PRAGMA <dbname>.journal_mode = WAL;"
-   "PRAGMA locking_mode = EXCLUSIVE;"
+   "PRAGMA <dbname>.locking_mode = EXCLUSIVE;"
    ""
    // project is a binary representation of an XML file.
    // it's in binary for speed.
@@ -58,7 +60,7 @@ static const char *ProjectFileSchema =
    // There is no limit to document blob size.
    // dict will be smallish, with an entry for each 
    // kind of field.
-   "CREATE TABLE IF NOT EXISTS project"
+   "CREATE TABLE IF NOT EXISTS <dbname>.project"
    "("
    "  id                   INTEGER PRIMARY KEY,"
    "  dict                 BLOB,"
@@ -78,7 +80,7 @@ static const char *ProjectFileSchema =
    // There is no limit to document blob size.
    // dict will be smallish, with an entry for each 
    // kind of field.
-   "CREATE TABLE IF NOT EXISTS autosave"
+   "CREATE TABLE IF NOT EXISTS <dbname>.autosave"
    "("
    "  id                   INTEGER PRIMARY KEY,"
    "  dict                 BLOB,"
@@ -87,7 +89,7 @@ static const char *ProjectFileSchema =
    ""
    // CREATE SQL tags
    // tags is not used (yet)
-   "CREATE TABLE IF NOT EXISTS tags"
+   "CREATE TABLE IF NOT EXISTS <dbname>.tags"
    "("
    "  name                 TEXT,"
    "  value                BLOB"
@@ -104,7 +106,7 @@ static const char *ProjectFileSchema =
    // blockID is a 64 bit number.
    //
    // summin to summary64K are summaries at 3 distance scales.
-   "CREATE TABLE IF NOT EXISTS sampleblocks"
+   "CREATE TABLE IF NOT EXISTS <dbname>.sampleblocks"
    "("
    "  blockid              INTEGER PRIMARY KEY AUTOINCREMENT,"
    "  sampleformat         INTEGER,"
@@ -609,7 +611,7 @@ bool ProjectFileIO::CheckVersion()
    // must be a new project file.
    if (wxStrtol<char **>(result, nullptr, 10) == 0)
    {
-      return InstallSchema();
+      return InstallSchema(db);
    }
 
    // Check for our application ID
@@ -653,19 +655,15 @@ bool ProjectFileIO::CheckVersion()
    return true;
 }
 
-bool ProjectFileIO::InstallSchema()
+bool ProjectFileIO::InstallSchema(sqlite3 *db, const char *dbname /* = "main" */)
 {
-   auto db = DB();
    int rc;
 
-   char sql[1024];
+   wxString sql;
-   sqlite3_snprintf(sizeof(sql),
+   sql.Printf(ProjectFileSchema, ProjectFileID, ProjectFileVersion);
-                    sql,
+   sql.Replace("<dbname>", dbname);
-                    ProjectFileSchema,
-                    ProjectFileID,
-                    ProjectFileVersion);
 
-   rc = sqlite3_exec(db, sql, nullptr, nullptr, nullptr);
+   rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
    if (rc != SQLITE_OK)
    {
       SetDBError(
@@ -688,12 +686,12 @@ bool ProjectFileIO::UpgradeSchema()
 // An SQLite function that takes a blockid and looks it up in a set of
 // blockids captured during project load.  If the blockid isn't found
 // in the set, it will be deleted.
-void ProjectFileIO::isorphan(sqlite3_context *context, int argc, sqlite3_value **argv)
+void ProjectFileIO::InSet(sqlite3_context *context, int argc, sqlite3_value **argv)
 {
    BlockIDs *blockids = (BlockIDs *) sqlite3_user_data(context);
    SampleBlockID blockid = sqlite3_value_int64(argv[0]);
 
-   sqlite3_result_int(context, blockids->find(blockid) == blockids->end());
+   sqlite3_result_int(context, blockids->find(blockid) != blockids->end());
 }
 
 bool ProjectFileIO::CheckForOrphans(BlockIDs &blockids)
@@ -701,18 +699,25 @@ bool ProjectFileIO::CheckForOrphans(BlockIDs &blockids)
    auto db = DB();
    int rc;
 
-   // Add our function that will verify blockid against the set of valid blockids
+   auto cleanup = finally([&]
-   rc = sqlite3_create_function(db, "isorphan", 1, SQLITE_UTF8, &blockids, isorphan, nullptr, nullptr);
+   {
+      // Remove our function, whether it was successfully defined or not.
+      sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, nullptr, nullptr, nullptr);
+   });
+
+   // Add the function used to verify each rows blockid against the set of active blockids
+   rc = sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, &blockids, InSet, nullptr, nullptr);
    if (rc != SQLITE_OK)
    {
-      //asdfasdf
+      wxLogDebug(wxT("Unable to add 'inset' function"));
       return false;
    }
 
    // Delete all rows that are orphaned
-   rc = sqlite3_exec(db, "DELETE FROM sampleblocks WHERE isorphan(blockid);", nullptr, nullptr, nullptr);
+   rc = sqlite3_exec(db, "DELETE FROM sampleblocks WHERE NOT inset(blockid);", nullptr, nullptr, nullptr);
    if (rc != SQLITE_OK)
    {
+      wxLogWarning(XO("Cleanup of orphan blocks failed").Translation());
       return false;
    }
 
@@ -745,32 +750,81 @@ static int progress_callback(void *data)
 
 sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath)
 {
+   // Get access to the active tracklist
+   auto pProject = mpProject.lock();
+   if (!pProject)
+   {
+      return nullptr;
+   }
+   auto &tracklist = TrackList::Get(*pProject);
+
+   // Collect all active blockids
+   BlockIDs blockids;
+   for (auto wt : tracklist.Any<const WaveTrack>())
+   {
+      // Scan all clips within current track
+      for (const auto &clip : wt->GetAllClips())
+      {
+         // Scan all blockfiles within current clip
+         auto blocks = clip->GetSequenceBlockArray();
+         for (const auto &block : *blocks)
+         {
+            blockids.insert(block.sb->GetBlockID());
+         }
+      }
+   }
+
    auto db = DB();
+   sqlite3 *destdb = nullptr;
+   bool success = false;
    int rc;
    ProgressResult res = ProgressResult::Success;
 
-   sqlite3_stmt *stmt = nullptr;
+   // Cleanup in case things go awry
    auto cleanup = finally([&]
    {
-      if (stmt)
+      // Detach the destination database, whether it was successfully attached or not
+      sqlite3_exec(db, "DETACH DATABASE dest;", nullptr, nullptr, nullptr);
+
+      // Remove our function, whether it was successfully defined or not.
+      sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, nullptr, nullptr, nullptr);
+
+      if (!success)
       {
-         sqlite3_finalize(stmt);
+         sqlite3_close(destdb);
+
+         wxRemoveFile(destpath);
       }
    });
 
-   rc = sqlite3_prepare_v2(db, "VACUUM INTO ?;", -1, &stmt, 0);
+   // Add the function used to verify each rows blockid against the set of active blockids
+   rc = sqlite3_create_function(db, "inset", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, &blockids, InSet, nullptr, nullptr);
    if (rc != SQLITE_OK)
    {
       SetDBError(
-         XO("Unable to prepare project file command")
+         XO("Unable to add 'inset' function")
       );
       return nullptr;
    }
 
-   rc = sqlite3_bind_text(stmt, 1, destpath.mb_str().data(), destpath.mb_str().length(), SQLITE_STATIC);
+   // Attach the destination database 
+   wxString sql;
+   sql.Printf("ATTACH DATABASE '%s' AS dest;", destpath);
+
+   rc = sqlite3_exec(db, sql.mb_str().data(), nullptr, nullptr, nullptr);
    if (rc != SQLITE_OK)
    {
-      THROW_INCONSISTENCY_EXCEPTION;
+      SetDBError(
+         XO("Unable to attach destination database")
+      );
+      return nullptr;
+   }
+
+   // Install our schema into the new database
+   if (!InstallSchema(db, "dest"))
+   {
+      // Message already set
+      return nullptr;
    }
 
    {
@@ -780,27 +834,37 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath)
 
       sqlite3_progress_handler(db, 10, progress_callback, &progress);
 
-      rc = sqlite3_step(stmt);
+      rc = sqlite3_exec(db,
+                        "INSERT INTO dest.tags"
+                        "       SELECT * FROM main.tags;",
+                        nullptr,
+                        nullptr,
+                        nullptr);
+      if (rc == SQLITE_OK)
+      {
+         rc = sqlite3_exec(db,
+                           "INSERT INTO dest.sampleblocks"
+                           "       SELECT * FROM main.sampleblocks"
+                           "       WHERE inset(blockid);",
+                           nullptr,
+                           nullptr,
+                           nullptr);
+      }
 
       sqlite3_progress_handler(db, 0, nullptr, nullptr);
    }
 
-   sqlite3_finalize(stmt);
+   // Copy failed
-   stmt = nullptr;
+   if (rc != SQLITE_OK)
-
-   // VACUUMing failed
-   if (rc != SQLITE_DONE)
    {
       SetDBError(
-         XO("Project file copy failed")
+         XO("Failed to copy project file")
       );
 
-      wxRemoveFile(destpath);
-
       return nullptr;
    }
 
-   sqlite3 *destdb = nullptr;
+   // Open the newly created database
    rc = sqlite3_open(destpath, &destdb);
    if (rc != SQLITE_OK)
    {
@@ -808,13 +872,12 @@ sqlite3 *ProjectFileIO::CopyTo(const FilePath &destpath)
          XO("Failed to open copy of project file")
       );
 
-      sqlite3_close(destdb);
-
-      wxRemoveFile(destpath);
-
       return nullptr;
    }
 
+   // Tell cleanup everything is good to go
+   success = true;
+
    return destdb;
 }
 
@@ -1391,14 +1454,6 @@ bool ProjectFileIO::SaveProject(const FilePath &fileName)
       return false;
    }
 
-   // We need to remove the autosave info from the file since it is now
-   // clean and unmodified.  Otherwise, it would be considered "recovered"
-   // when next opened.
-   if (!AutoSaveDelete())
-   {
-      return false;
-   }
-
    // Reaching this point defines success and all the rest are no-fail
    // operations:
 
@@ -1453,14 +1508,6 @@ bool ProjectFileIO::SaveCopy(const FilePath& fileName)
       return false;
    }
 
-   // We need to remove the autosave info from the new DB since it is now
-   // clean and unmodified.  Otherwise, it would be considered "recovered"
-   // when next opened.
-   if (!AutoSaveDelete(db))
-   {
-      return false;
-   }
-
    // Tell the finally block to behave
    success = true;
 

src/ProjectFileIO.h

@@ -150,15 +150,17 @@ private:
    bool GetBlob(const char *sql, wxMemoryBuffer &buffer);
 
    bool CheckVersion();
-   bool InstallSchema();
+   bool InstallSchema(sqlite3 *db, const char *dbname = "main");
    bool UpgradeSchema();
 
    // Write project or autosave XML (binary) documents
    bool WriteDoc(const char *table, const ProjectSerializer &autosave, sqlite3 *db = nullptr);
 
-   // Checks for orphan blocks.  This will go away at a future date
+   // Application defined function to verify blockid exists is in set of blockids
    using BlockIDs = std::set<SampleBlockID>;
-   static void isorphan(sqlite3_context *context, int argc, sqlite3_value **argv);
+   static void InSet(sqlite3_context *context, int argc, sqlite3_value **argv);
+
+   // Checks for orphan blocks.  This will go away at a future date
    bool CheckForOrphans(BlockIDs &blockids);
 
    // Return a database connection if successful, which caller must close

src/ProjectFileManager.cpp

@@ -599,10 +599,9 @@ bool ProjectFileManager::SaveCopy(const FilePath &fileName /* = wxT("") */)
 
    if (!projectFileIO.SaveCopy(filename.GetFullPath()))
    {
-      // Overwrite disallowed. The destination project is open in another window.
       AudacityMessageDialog m(
          nullptr,
-         XO("The project will not saved because the selected project is open in another window.\nPlease try again and select an original name."),
+         XO("The project was not saved.  See Help->Diagnostics->Show Log for more info."),
          XO("Error Saving Project"),
          wxOK | wxICON_ERROR);
 

src/import/ImportAUP.cpp

@@ -705,7 +705,7 @@ bool AUPImportFileHandle::HandleProject(XMLTagHandler *&handler)
          if (projName.empty())
          {
             AudacityMessageBox(
-               XO("Couldn't find the project data folder: \"%s\"").Format(*value),
+               XO("Couldn't find the project data folder: \"%s\"").Format(value),
                XO("Error Opening Project"),
                wxOK | wxCENTRE,
                &window);