diff --git a/src/AutoRecovery.cpp b/src/AutoRecovery.cpp index 381bee079..e431c0cf3 100644 --- a/src/AutoRecovery.cpp +++ b/src/AutoRecovery.cpp @@ -267,7 +267,8 @@ bool AutoSaveFile::DictChanged() const return mDictChanged; } -wxString AutoSaveFile::Decode(const wxMemoryBuffer &buffer) +// See ProjectFileIO::CheckForOrphans() for explanation of the blockids arg +wxString AutoSaveFile::Decode(const wxMemoryBuffer &buffer, BlockIDs &blockids) { wxMemoryInputStream in(buffer.GetData(), buffer.GetDataLen()); @@ -297,205 +298,218 @@ wxString AutoSaveFile::Decode(const wxMemoryBuffer &buffer) switch (mCharSize) { - case 1: - str.assignFromUTF8(in, len); - break; + case 1: + str.assignFromUTF8(in, len); + break; - case 2: - str.assignFromUTF16((wxChar16 *) in, len / 2); - break; + case 2: + str.assignFromUTF16((wxChar16 *) in, len / 2); + break; - case 4: - { - str = wxU32CharBuffer::CreateNonOwned((wxChar32 *) in, len / 4); - } - break; + case 4: + str = wxU32CharBuffer::CreateNonOwned((wxChar32 *) in, len / 4); + break; - default: - wxASSERT_MSG(false, wxT("Characters size not 1, 2, or 4")); + default: + wxASSERT_MSG(false, wxT("Characters size not 1, 2, or 4")); + break; } return str; }; - try { while (!in.Eof()) + try { - short id; - - switch (in.GetC()) + while (!in.Eof()) { - case FT_Push: + short id; + + switch (in.GetC()) { - mIdStack.push_back(mIds); - mIds.clear(); + case FT_Push: + { + mIdStack.push_back(mIds); + mIds.clear(); + } + break; + + case FT_Pop: + { + mIds = mIdStack.back(); + mIdStack.pop_back(); + } + break; + + case FT_Name: + { + short len; + + in.Read(&id, sizeof(id)); + in.Read(&len, sizeof(len)); + bytes.reserve(len); + in.Read(bytes.data(), len); + + mIds[id] = Convert(bytes.data(), len); + } + break; + + case FT_StartTag: + { + in.Read(&id, sizeof(id)); + + out.StartTag(Lookup(id)); + } + break; + + case FT_EndTag: + { + in.Read(&id, sizeof(id)); + + out.EndTag(Lookup(id)); + } + break; + + case FT_String: + { + int len; + + in.Read(&id, sizeof(id)); + in.Read(&len, sizeof(len)); + bytes.reserve(len); + in.Read(bytes.data(), len); + + out.WriteAttr(Lookup(id), Convert(bytes.data(), len)); + } + break; + + case FT_Float: + { + float val; + int dig; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + in.Read(&dig, sizeof(dig)); + + out.WriteAttr(Lookup(id), val, dig); + } + break; + + case FT_Double: + { + double val; + int dig; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + in.Read(&dig, sizeof(dig)); + + out.WriteAttr(Lookup(id), val, dig); + } + break; + + case FT_Int: + { + int val; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + + out.WriteAttr(Lookup(id), val); + } + break; + + case FT_Bool: + { + bool val; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + + out.WriteAttr(Lookup(id), val); + } + break; + + case FT_Long: + { + long val; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + + out.WriteAttr(Lookup(id), val); + } + break; + + case FT_LongLong: + { + long long val; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + + // Look for and save the "blockid" values to support orphan + // block checking. This should be removed once autosave and + // related blocks become part of the same transaction. + const wxString &name = Lookup(id); + if (name.IsSameAs(wxT("blockid"))) + { + blockids.insert(val); + } + + out.WriteAttr(Lookup(id), val); + } + break; + + case FT_SizeT: + { + size_t val; + + in.Read(&id, sizeof(id)); + in.Read(&val, sizeof(val)); + + out.WriteAttr(Lookup(id), val); + } + break; + + case FT_Data: + { + int len; + + in.Read(&len, sizeof(len)); + bytes.reserve(len); + in.Read(bytes.data(), len); + + out.WriteData(Convert(bytes.data(), len)); + } + break; + + case FT_Raw: + { + int len; + + in.Read(&len, sizeof(len)); + bytes.reserve(len); + in.Read(bytes.data(), len); + + out.Write(Convert(bytes.data(), len)); + } + break; + + case FT_CharSize: + { + in.Read(&mCharSize, sizeof(mCharSize)); + } + break; + + default: + wxASSERT(true); + break; } - break; - - case FT_Pop: - { - mIds = mIdStack.back(); - mIdStack.pop_back(); - } - break; - - case FT_Name: - { - short len; - - in.Read(&id, sizeof(id)); - in.Read(&len, sizeof(len)); - bytes.reserve(len); - in.Read(bytes.data(), len); - - mIds[id] = Convert(bytes.data(), len); - } - break; - - case FT_StartTag: - { - in.Read(&id, sizeof(id)); - - out.StartTag(Lookup(id)); - } - break; - - case FT_EndTag: - { - in.Read(&id, sizeof(id)); - - out.EndTag(Lookup(id)); - } - break; - - case FT_String: - { - int len; - - in.Read(&id, sizeof(id)); - in.Read(&len, sizeof(len)); - bytes.reserve(len); - in.Read(bytes.data(), len); - - out.WriteAttr(Lookup(id), Convert(bytes.data(), len)); - } - break; - - case FT_Float: - { - float val; - int dig; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - in.Read(&dig, sizeof(dig)); - - out.WriteAttr(Lookup(id), val, dig); - } - break; - - case FT_Double: - { - double val; - int dig; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - in.Read(&dig, sizeof(dig)); - - out.WriteAttr(Lookup(id), val, dig); - } - break; - - case FT_Int: - { - int val; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - - out.WriteAttr(Lookup(id), val); - } - break; - - case FT_Bool: - { - bool val; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - - out.WriteAttr(Lookup(id), val); - } - break; - - case FT_Long: - { - long val; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - - out.WriteAttr(Lookup(id), val); - } - break; - - case FT_LongLong: - { - long long val; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - - out.WriteAttr(Lookup(id), val); - } - break; - - case FT_SizeT: - { - size_t val; - - in.Read(&id, sizeof(id)); - in.Read(&val, sizeof(val)); - - out.WriteAttr(Lookup(id), val); - } - break; - - case FT_Data: - { - int len; - - in.Read(&len, sizeof(len)); - bytes.reserve(len); - in.Read(bytes.data(), len); - - out.WriteData(Convert(bytes.data(), len)); - } - break; - - case FT_Raw: - { - int len; - - in.Read(&len, sizeof(len)); - bytes.reserve(len); - in.Read(bytes.data(), len); - - out.Write(Convert(bytes.data(), len)); - } - break; - - case FT_CharSize: - { - in.Read(&mCharSize, sizeof(mCharSize)); - } - break; - - default: - wxASSERT(true); - break; } - } } catch( const Error& ) { + } + catch( const Error& ) + { // Autosave was corrupt, or platform differences in size or endianness // were not well canonicalized return {}; diff --git a/src/AutoRecovery.h b/src/AutoRecovery.h index a553dbef2..303077dbd 100644 --- a/src/AutoRecovery.h +++ b/src/AutoRecovery.h @@ -18,6 +18,12 @@ #include #include "audacity/Types.h" +// From SampleBlock.h +using SampleBlockID = long long; + +// From ProjectFileiIO.h +using BlockIDs = std::set; + /// /// AutoSaveFile /// @@ -62,7 +68,7 @@ public: bool DictChanged() const; // Returns empty string if decoding fails - static wxString Decode(const wxMemoryBuffer &buffer); + static wxString Decode(const wxMemoryBuffer &buffer, BlockIDs &blockids); private: void WriteName(const wxString & name); diff --git a/src/ProjectFileIO.cpp b/src/ProjectFileIO.cpp index 35091fe79..63eb27244 100644 --- a/src/ProjectFileIO.cpp +++ b/src/ProjectFileIO.cpp @@ -46,14 +46,23 @@ static const char *ProjectFileSchema = "PRAGMA journal_mode = WAL;" "PRAGMA locking_mode = EXCLUSIVE;" "" - // CREATE SQL project - // doc is a variable sized XML text string. - // it is the former Audacity .aup file - // One instance only. + // project is a binary representation of an XML file. + // it's in binary for speed. + // One instance only. id is always 1. + // dict is a dictionary of fieldnames. + // doc is the binary representation of the XML + // in the doc, fieldnames are replaced by 2 byte dictionary + // index numbers. + // This is all opaque to SQLite. It just sees two + // big binary blobs. + // There is no limit to document blob size. + // dict will be smallish, with an entry for each + // kind of field. "CREATE TABLE IF NOT EXISTS project" "(" " id INTEGER PRIMARY KEY," - " doc TEXT" + " dict BLOB," + " doc BLOB" ");" "" // CREATE SQL autosave @@ -438,7 +447,7 @@ bool ProjectFileIO::TransactionCommit(const wxString &name) char* errmsg = nullptr; int rc = sqlite3_exec(DB(), - wxT("SAVEPOINT ") + name + wxT(";"), + wxT("RELEASE ") + name + wxT(";"), nullptr, nullptr, &errmsg); @@ -459,7 +468,7 @@ bool ProjectFileIO::TransactionRollback(const wxString &name) char* errmsg = nullptr; int rc = sqlite3_exec(DB(), - wxT("RELEASE ") + name + wxT(";"), + wxT("ROLLBACK TO ") + name + wxT(";"), nullptr, nullptr, &errmsg); @@ -503,6 +512,8 @@ int ProjectFileIO::Exec(const char *query, ExecCB callback, wxString *result) bool ProjectFileIO::GetValue(const char *sql, wxString &result) { + result.clear(); + auto getresult = [](wxString *result, int cols, char **vals, char **names) { if (cols == 1 && vals[0]) @@ -528,6 +539,8 @@ bool ProjectFileIO::GetBlob(const char *sql, wxMemoryBuffer &buffer) auto db = DB(); int rc; + buffer.Clear(); + sqlite3_stmt *stmt = nullptr; auto cleanup = finally([&] { @@ -566,7 +579,6 @@ bool ProjectFileIO::GetBlob(const char *sql, wxMemoryBuffer &buffer) const void *blob = sqlite3_column_blob(stmt, 0); int size = sqlite3_column_bytes(stmt, 0); - buffer.Clear(); buffer.AppendData(blob, size); return true; @@ -661,43 +673,41 @@ bool ProjectFileIO::UpgradeSchema() return true; } -void ProjectFileIO::LoadedBlock(int64_t sbid) +// The orphan block handling should be removed once autosave and related +// blocks become part of the same transaction. + +// An SQLite function that takes a blockid and looks it up in a set of +// blockids captured during project load. If the blockid isn't found +// in the set, it will be deleted. +void ProjectFileIO::isorphan(sqlite3_context *context, int argc, sqlite3_value **argv) { - if (sbid > mHighestBlockID) - { - mHighestBlockID = sbid; - } + BlockIDs *blockids = (BlockIDs *) sqlite3_user_data(context); + SampleBlockID blockid = sqlite3_value_int64(argv[0]); + + sqlite3_result_int(context, blockids->find(blockid) == blockids->end()); } -bool ProjectFileIO::CheckForOrphans() +bool ProjectFileIO::CheckForOrphans(BlockIDs &blockids) { auto db = DB(); + int rc; - char sql[256]; - sqlite3_snprintf(sizeof(sql), - sql, - "DELETE FROM sampleblocks WHERE blockid > %lld", - mHighestBlockID); - - char *errmsg = nullptr; - - int rc = sqlite3_exec(db, sql, nullptr, nullptr, &errmsg); - - if (errmsg) - { - mLibraryError = Verbatim(wxString(errmsg)); - sqlite3_free(errmsg); - } - + // Add our function that will verify blockid against the set of valid blockids + rc = sqlite3_create_function(db, "isorphan", 1, SQLITE_UTF8, &blockids, isorphan, nullptr, nullptr); if (rc != SQLITE_OK) { - SetDBError( - XO("Failed to delete orphaned blocks") - ); - + //asdfasdf return false; } + // Delete all rows that are orphaned + rc = sqlite3_exec(db, "DELETE FROM sampleblocks WHERE isorphan(blockid);", nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) + { + return false; + } + + // Mark the project recovered if we deleted any rows int changes = sqlite3_changes(db); if (changes > 0) { @@ -1093,23 +1103,45 @@ bool ProjectFileIO::AutoSave(const WaveTrackArray *tracks) WriteXMLHeader(autosave); WriteXML(autosave, tracks); - return AutoSave(autosave); + if (WriteDoc("autosave", autosave)) + { + mModified = true; + return true; + } + + return false; } -bool ProjectFileIO::AutoSave(const AutoSaveFile &autosave) +bool ProjectFileIO::AutoSaveDelete() { auto db = DB(); int rc; - mModified = true; + rc = sqlite3_exec(db, "DELETE FROM autosave;", nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) + { + SetDBError( + XO("Failed to remove the autosave information from the project file.") + ); + return false; + } + + return true; +} + +bool ProjectFileIO::WriteDoc(const char *table, const AutoSaveFile &autosave) +{ + auto db = DB(); + int rc; // For now, we always use an ID of 1. This will replace the previously // writen row every time. char sql[256]; sqlite3_snprintf(sizeof(sql), sql, - "INSERT INTO autosave(id, dict, doc) VALUES(1, ?1, ?2)" + "INSERT INTO %s(id, dict, doc) VALUES(1, ?1, ?2)" " ON CONFLICT(id) DO UPDATE SET %sdoc = ?2;", + table, autosave.DictChanged() ? "dict = ?1, " : ""); sqlite3_stmt *stmt = nullptr; @@ -1140,7 +1172,9 @@ bool ProjectFileIO::AutoSave(const AutoSaveFile &autosave) sqlite3_bind_blob(stmt, 1, dict.GetData(), dict.GetDataLen(), SQLITE_STATIC) || sqlite3_bind_blob(stmt, 2, data.GetData(), data.GetDataLen(), SQLITE_STATIC) ) + { THROW_INCONSISTENCY_EXCEPTION; + } rc = sqlite3_step(stmt); if (rc != SQLITE_DONE) @@ -1154,23 +1188,6 @@ bool ProjectFileIO::AutoSave(const AutoSaveFile &autosave) return true; } -bool ProjectFileIO::AutoSaveDelete() -{ - auto db = DB(); - int rc; - - rc = sqlite3_exec(db, "DELETE FROM autosave;", nullptr, nullptr, nullptr); - if (rc != SQLITE_OK) - { - SetDBError( - XO("Failed to remove the autosave information from the project file.") - ); - return false; - } - - return true; -} - bool ProjectFileIO::LoadProject(const FilePath &fileName) { bool success = false; @@ -1194,41 +1211,51 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName) return false; } - // Get the XML document...either from the project or autosave - wxString doc; + BlockIDs blockids; + wxString autosave; + wxString project; + wxMemoryBuffer buffer; // Get the autosave doc, if any - bool wasAutosave = false; - wxMemoryBuffer buffer; + if (!GetBlob("SELECT dict || doc FROM project WHERE id = 1;", buffer)) + { + // Error already set + return false; + } + if (buffer.GetDataLen() > 0) + { + project = AutoSaveFile::Decode(buffer, blockids); + } + if (!GetBlob("SELECT dict || doc FROM autosave WHERE id = 1;", buffer)) { // Error already set return false; } - if (buffer.GetDataLen() > 0) { - doc = AutoSaveFile::Decode(buffer); - wasAutosave = true; + autosave = AutoSaveFile::Decode(buffer, blockids); } - // Otherwise, get the project doc - else if (!GetValue("SELECT doc FROM project;", doc)) + + // Should this be an error??? + if (project.empty() && autosave.empty()) { - // Error already set + SetError(XO("Unable to load project or autosave documents")); return false; } - if (doc.empty()) + // Check for orphans blocks...set mRecovered if any deleted + if (blockids.size() > 0) { - return false; + if (!CheckForOrphans(blockids)) + { + return false; + } } XMLFileReader xmlFile; - // Reset the highest blockid - mHighestBlockID = 0; - - success = xmlFile.ParseString(this, doc); + success = xmlFile.ParseString(this, autosave.empty() ? project : autosave); if (!success) { SetError( @@ -1238,18 +1265,13 @@ bool ProjectFileIO::LoadProject(const FilePath &fileName) return false; } - // Remember if it was recovered or not - mRecovered = wasAutosave; - - // Check for orphans blocks...set mRecovered if any deleted - if (mHighestBlockID > 0) + // Remember if we used autosave or not + if (!autosave.empty()) { - if (!CheckForOrphans()) - { - return false; - } + mRecovered = true; } + // Mark the project modified if we recovered it if (mRecovered) { mModified = true; @@ -1277,9 +1299,7 @@ bool ProjectFileIO::SaveProject(const FilePath &fileName) bool wasTemp = false; bool success = false; - // Should probably simplify all of the following by using renames. But, one - // benefit of using CopyTo() for new file saves, is that it will be VACUUMED - // at the same time. + // Should probably simplify all of the following by using renames. auto restore = finally([&] { @@ -1336,65 +1356,22 @@ bool ProjectFileIO::SaveProject(const FilePath &fileName) auto db = DB(); int rc; - XMLStringWriter doc; + AutoSaveFile doc; WriteXMLHeader(doc); WriteXML(doc); - // Always use an ID of 1. This will replace any existing row. - char sql[256]; - sqlite3_snprintf(sizeof(sql), - sql, - "INSERT INTO project(id, doc) VALUES(1, ?1)" - " ON CONFLICT(id) DO UPDATE SET doc = ?1;"); - - + if (!WriteDoc("project", doc)) { - sqlite3_stmt* stmt = nullptr; - - auto finalize = finally([&] - { - if (stmt) - { - // This will free the statement - sqlite3_finalize(stmt); - } - }); - - rc = sqlite3_prepare_v2(db, sql, -1, &stmt, 0); - if (rc != SQLITE_OK) - { - SetDBError( - XO("Unable to prepare project file command:\n\n%s").Format(sql) - ); - return false; - } - - // BIND SQL project - rc = sqlite3_bind_text(stmt, 1, doc, -1, SQLITE_STATIC); - if (rc != SQLITE_OK) - { - SetDBError( - XO("Unable to bind to project file document.") - ); - return false; - } - - rc = sqlite3_step(stmt); - if (rc != SQLITE_DONE) - { - SetDBError( - XO("Failed to save project file information.") - ); - return false; - } + return false; } // We need to remove the autosave info from the file since it is now // clean and unmodified. Otherwise, it would be considered "recovered" // when next opened. - - if ( !AutoSaveDelete() ) + if (!AutoSaveDelete()) + { return false; + } // Reaching this point defines success and all the rest are no-fail // operations: @@ -1584,3 +1561,41 @@ bool ProjectFileIO::ShouldBypass() return mTemporary && mBypass; } +AutoCommitTransaction::AutoCommitTransaction(ProjectFileIO &projectFileIO, + const char *name) +: mIO(projectFileIO), + mName(name) +{ + mInTrans = mIO.TransactionStart(mName); + // Must throw +} + +AutoCommitTransaction::~AutoCommitTransaction() +{ + if (mInTrans) + { + // Can't check return status...should probably throw an exception here + if (!Commit()) + { + // must throw + } + } +} + +bool AutoCommitTransaction::Commit() +{ + wxASSERT(mInTrans); + + mInTrans = !mIO.TransactionCommit(mName); + + return mInTrans; +} + +bool AutoCommitTransaction::Rollback() +{ + wxASSERT(mInTrans); + + mInTrans = !mIO.TransactionCommit(mName); + + return mInTrans; +} diff --git a/src/ProjectFileIO.h b/src/ProjectFileIO.h index 92e0ebc1a..0d022d85e 100644 --- a/src/ProjectFileIO.h +++ b/src/ProjectFileIO.h @@ -17,6 +17,8 @@ Paul Licameli split from AudacityProject.h #include "xml/XMLTagHandler.h" // to inherit struct sqlite3; +struct sqlite3_context; +struct sqlite3_value; class AudacityProject; class AutoCommitTransaction; @@ -26,6 +28,9 @@ class WaveTrack; using WaveTrackArray = std::vector < std::shared_ptr < WaveTrack > >; +// From SampleBlock.h +using SampleBlockID = long long; + ///\brief Object associated with a project that manages reading and writing /// of Audacity project file formats, and autosave class ProjectFileIO final @@ -67,7 +72,6 @@ public: void Reset(); bool AutoSave(const WaveTrackArray *tracks = nullptr); - bool AutoSave(const AutoSaveFile &autosave); bool AutoSaveDelete(); bool LoadProject(const FilePath &fileName); @@ -98,8 +102,6 @@ public: void Bypass(bool bypass); bool ShouldBypass(); - void LoadedBlock(int64_t blockID); - private: // XMLTagHandler callback methods bool HandleXMLTag(const wxChar *tag, const wxChar **attrs) override; @@ -148,8 +150,13 @@ private: bool InstallSchema(); bool UpgradeSchema(); + // Write project or autosave XML (binary) documents + bool WriteDoc(const char *table, const AutoSaveFile &autosave); + // Checks for orphan blocks. This will go away at a future date - bool CheckForOrphans(); + using BlockIDs = std::set; + static void isorphan(sqlite3_context *context, int argc, sqlite3_value **argv); + bool CheckForOrphans(BlockIDs &blockids); // Return a database connection if successful, which caller must close sqlite3 *CopyTo(const FilePath &destpath); @@ -186,6 +193,22 @@ private: int64_t mHighestBlockID; friend SqliteSampleBlock; + friend AutoCommitTransaction; +}; + +class AutoCommitTransaction +{ +public: + AutoCommitTransaction(ProjectFileIO &projectFileIO, const char *name); + ~AutoCommitTransaction(); + + bool Commit(); + bool Rollback(); + +private: + ProjectFileIO &mIO; + bool mInTrans; + wxString mName; }; class wxTopLevelWindow; diff --git a/src/SampleBlock.h b/src/SampleBlock.h index 55d1037fb..466633b4a 100644 --- a/src/SampleBlock.h +++ b/src/SampleBlock.h @@ -25,7 +25,6 @@ using SampleBlockFactoryPtr = std::shared_ptr; using SampleBlockFactoryFactory = std::function< SampleBlockFactoryPtr( AudacityProject& ) >; -//using SampleBlockID = sqlite3_int64; // Trying not to depend on sqlite headers using SampleBlockID = long long; class MinMaxRMS diff --git a/src/SqliteSampleBlock.cpp b/src/SqliteSampleBlock.cpp index 04b4e0e46..ae1760dc6 100644 --- a/src/SqliteSampleBlock.cpp +++ b/src/SqliteSampleBlock.cpp @@ -186,11 +186,6 @@ SampleBlockPtr SqliteSampleBlockFactory::DoCreateFromXML( { // This may throw sb->Load((SampleBlockID) nValue); - - // Tell the IO manager the blockid we just loaded so it can track - // the highest one encountered. - mpIO->LoadedBlock(nValue); - found++; } else if (wxStrcmp(attr, wxT("samplecount")) == 0)