From 4f4472b824b0ab2fb0c7c487a03c20959ac620df Mon Sep 17 00:00:00 2001 From: Fred Gleason Date: Sat, 30 Aug 2025 11:05:02 -0400 Subject: [PATCH] 2025-08-30 Fred Gleason * Added a 'LogDropboxProcessing=' directive to the '[Debugging]' section of rd.conf(5). Signed-off-by: Fred Gleason --- ChangeLog | 3 + conf/rd.conf-sample | 8 ++ lib/rdconfig.cpp | 22 +++++- lib/rdconfig.h | 4 +- utils/rdimport/Makefile.am | 3 +- utils/rdimport/dropboxlist.cpp | 42 ++++++++++ utils/rdimport/dropboxlist.h | 40 ++++++++++ utils/rdimport/rdimport.cpp | 137 +++++++++++++++++++++++++-------- utils/rdimport/rdimport.h | 12 ++- 9 files changed, 231 insertions(+), 40 deletions(-) create mode 100644 utils/rdimport/dropboxlist.cpp create mode 100644 utils/rdimport/dropboxlist.h diff --git a/ChangeLog b/ChangeLog index 67f038ca..19838348 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25039,3 +25039,6 @@ 2025-08-29 Fred Gleason * Added an item to update the ChangeSummary on the Rivendell wiki to section three of 'docs/misc/rivendell_release_checklist.txt'. +2025-08-30 Fred Gleason + * Added a 'LogDropboxProcessing=' directive to the '[Debugging]' + section of rd.conf(5). diff --git a/conf/rd.conf-sample b/conf/rd.conf-sample index 9f8d0125..a30a64f4 100644 --- a/conf/rd.conf-sample +++ b/conf/rd.conf-sample @@ -281,3 +281,11 @@ LogSqlQueries= ; generate a useful error message in ; RDAdmin->ManageHosts->PyPADInstances->ErrorLog. KillPypadAfterJsonError=No + +; Log the internal processing steps used by rdimport(1) when processing the +; specified dropbox instances. Take a comma-delimited list of dropbox IDs, +; which can be found in the 'ID' column of the dropbox list at +; RDAdmin->ManageHosts->[hostname]->Dropboxes. The log messages are sent +; DEBUG priority level. +; LogDropboxProcessing=,, +;LogDropboxProcessing=27,52,12 diff --git a/lib/rdconfig.cpp b/lib/rdconfig.cpp index 5007da64..c566971b 100644 --- a/lib/rdconfig.cpp +++ b/lib/rdconfig.cpp @@ -2,7 +2,7 @@ // // A container class for a Rivendell Base Configuration // -// (C) Copyright 2002-2024 Fred Gleason +// (C) Copyright 2002-2025 Fred Gleason // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License version 2 as @@ -441,6 +441,12 @@ bool RDConfig::killPypadAfterJsonError() const } +QList RDConfig::logDropboxProcessingIds() const +{ + return conf_log_dropbox_processing_ids; +} + + int RDConfig::logSqlQueriesLevel() const { return conf_log_sql_queries_level; @@ -704,6 +710,19 @@ bool RDConfig::load() &conf_log_sql_queries); conf_kill_pypad_after_json_error= profile->boolValue("Debugging","KillPypadAfterJsonError"); + + QStringList f0=profile->stringValue("Debugging","LogDropboxProcessing"). + split(",",QString::KeepEmptyParts); + for(int i=0;i0)) { + conf_log_dropbox_processing_ids.push_back(id); + } + else { + fprintf(stderr,"WARNING: invalid dropbox ID \"%s\" specified in LogDropboxProcessing=\n",f0.at(i).toUtf8().constData()); + } + } conf_meter_base_port= profile->intValue("Hacks","MeterPortBaseNumber",RD_DEFAULT_METER_SOCKET_BASE_UDP_PORT); conf_meter_port_range= @@ -847,6 +866,7 @@ void RDConfig::clear() conf_log_sql_queries=false; conf_log_sql_queries_level=LOG_DEBUG; conf_kill_pypad_after_json_error=false; + conf_log_dropbox_processing_ids.clear(); conf_lock_rdairplay_memory=false; conf_meter_base_port=RD_DEFAULT_METER_SOCKET_BASE_UDP_PORT; conf_meter_port_range=RD_METER_SOCKET_PORT_RANGE; diff --git a/lib/rdconfig.h b/lib/rdconfig.h index c73321e7..e513211a 100644 --- a/lib/rdconfig.h +++ b/lib/rdconfig.h @@ -2,7 +2,7 @@ // // A container class for a Rivendell Base Configuration // -// (C) Copyright 2002-2024 Fred Gleason +// (C) Copyright 2002-2025 Fred Gleason // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License version 2 as @@ -113,6 +113,7 @@ class RDConfig bool logSqlQueries() const; int logSqlQueriesLevel() const; bool killPypadAfterJsonError() const; + QList logDropboxProcessingIds() const; bool enableMixerLogging() const; bool testOutputStreams() const; uid_t uid() const; @@ -195,6 +196,7 @@ class RDConfig int conf_log_log_refresh_level; bool conf_log_sql_queries; bool conf_kill_pypad_after_json_error; + QList conf_log_dropbox_processing_ids; int conf_log_sql_queries_level; bool conf_lock_rdairplay_memory; QString conf_save_webget_files_directory; diff --git a/utils/rdimport/Makefile.am b/utils/rdimport/Makefile.am index 1dbc847d..f9f6c15a 100644 --- a/utils/rdimport/Makefile.am +++ b/utils/rdimport/Makefile.am @@ -27,7 +27,8 @@ moc_%.cpp: %.h bin_PROGRAMS = rdimport -dist_rdimport_SOURCES = journal.cpp journal.h\ +dist_rdimport_SOURCES = dropboxlist.cpp dropboxlist.h\ + journal.cpp journal.h\ markerset.cpp markerset.h\ rdimport.cpp rdimport.h diff --git a/utils/rdimport/dropboxlist.cpp b/utils/rdimport/dropboxlist.cpp new file mode 100644 index 00000000..bcec7307 --- /dev/null +++ b/utils/rdimport/dropboxlist.cpp @@ -0,0 +1,42 @@ +// dropboxlist.cpp +// +// List of dropbox file entries +// +// (C) Copyright 2002-2025 Fred Gleason +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +// + +#include + +#include "dropboxlist.h" + +DropboxList::DropboxList(const QString &fname) +{ + QFile *file=new QFile(fname); + filename=fname; + size=file->size(); + pass=0; + checked=true; + failed=false; + delete file; +} + + +QString DropboxList::dump() const +{ + return QString::asprintf("%s: size: %ld pass: %d checked: %d failed: %d", + filename.toUtf8().constData(), + size,pass,checked,failed); +} diff --git a/utils/rdimport/dropboxlist.h b/utils/rdimport/dropboxlist.h new file mode 100644 index 00000000..9cf9b16b --- /dev/null +++ b/utils/rdimport/dropboxlist.h @@ -0,0 +1,40 @@ +// dropboxlist.h +// +// List of dropbox file entries +// +// (C) Copyright 2002-2025 Fred Gleason +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License version 2 as +// published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public +// License along with this program; if not, write to the Free Software +// Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +// + +#ifndef DROPBOXLIST_H +#define DROPBOXLIST_H + +#include + +#include + +class DropboxList { + public: + DropboxList(const QString &fname); + QString dump() const; + QString filename; + int64_t size; + unsigned pass; + bool checked; + bool failed; +}; + + +#endif // DROPBOXLIST_H diff --git a/utils/rdimport/rdimport.cpp b/utils/rdimport/rdimport.cpp index 371deede..5707cf4c 100644 --- a/utils/rdimport/rdimport.cpp +++ b/utils/rdimport/rdimport.cpp @@ -104,7 +104,8 @@ MainObject::MainObject(QObject *parent) import_dump_isci_xref=false; import_by_isci_program_code=""; import_by_isci=false; - + import_log_processing=false; + // // Open the Database // @@ -522,10 +523,17 @@ MainObject::MainObject(QObject *parent) ErrorExit(RDApplication::ExitInvalidOption); } if((import_cart_number>0)&&import_by_isci) { - Log(LOG_ERR,QString().sprintf("rdimport: --to-cart and --by-isci are mutually exclusive\n")); + Log(LOG_ERR,QString::asprintf("rdimport: --to-cart and --by-isci are mutually exclusive\n")); ErrorExit(RDApplication::ExitInvalidOption); } - + + // + // Enable Log Dropbox Processing + // + import_log_processing=(import_persistent_dropbox_id>0)&& + (rda->config()->logDropboxProcessingIds(). + contains(import_persistent_dropbox_id)); + import_cut_markers=new MarkerSet(); import_cut_markers->loadMarker(rda->cmdSwitch(),"cut"); import_talk_markers=new MarkerSet(); @@ -882,6 +890,18 @@ MainObject::MainObject(QObject *parent) if(import_string_year!=0) { Log(LOG_INFO,QString::asprintf(" Year set to: %d\n",import_string_year)); } + QString str=" LogDropboxProcessing = "; + QList ids=rda->config()->logDropboxProcessingIds(); + for(int i=0;i::const_iterator + for(std::list::const_iterator ci=import_dropbox_list.begin(); ci!=import_dropbox_list.end();ci++) { (*ci)->checked=false; @@ -1032,7 +1055,7 @@ void MainObject::RunDropBox() // // Take Out the Trash // - for(std::list::iterator + for(std::list::iterator ci=import_dropbox_list.begin(); ci!=import_dropbox_list.end();ci++) { if(!(*ci)->checked) { @@ -1050,11 +1073,13 @@ void MainObject::RunDropBox() void MainObject::ProcessFileEntry(const QString &entry) { + LogDropBox("ProcessFileEntry(\""+entry+"\")"); glob_t globbuf; int gflags=GLOB_MARK; if(entry=="-") { import_stdin_specified=true; + LogDropBox("STDIN path, no processing done"); return; } if(import_drop_box) { // Assume No Shell Preprocessing @@ -1066,6 +1091,7 @@ void MainObject::ProcessFileEntry(const QString &entry) entry.toUtf8().constData())); globfree(&globbuf); } + LogDropBox(QString::asprintf("found %lu matches",globbuf.gl_pathc)); for(size_t i=0;icloseWave(); import_failed_imports++; @@ -1259,7 +1286,7 @@ MainObject::Result MainObject::ImportFile(const QString &filename, *cartnum=wd->cartNumber(); } if(*cartnum==0) { - Log(LOG_WARNING,QString().sprintf(" File \"%s\" has no ISCI xreference entry, skipping...\n", + Log(LOG_WARNING,QString::asprintf(" File \"%s\" has no ISCI xreference entry, skipping...\n", RDGetBasePart(filename).toUtf8().constData())); wavefile->closeWave(); import_failed_imports++; @@ -1271,7 +1298,7 @@ MainObject::Result MainObject::ImportFile(const QString &filename, return MainObject::FileBad; } if(!effective_group->cartNumberValid(*cartnum)) { - Log(LOG_WARNING,QString().sprintf(" File \"%s\" cart number %06u is is not valid in group \"%s\", skipping...\n", + Log(LOG_WARNING,QString::asprintf(" File \"%s\" cart number %06u is is not valid in group \"%s\", skipping...\n", RDGetBasePart(filename).toUtf8().constData(), *cartnum, effective_group->name().toUtf8().constData())); @@ -1296,7 +1323,7 @@ MainObject::Result MainObject::ImportFile(const QString &filename, *cartnum=effective_group->nextFreeCart(); } if(*cartnum==0) { - Log(LOG_ERR,QString().sprintf("rdimport: no free carts available in specified group\n")); + Log(LOG_ERR,QString::asprintf("rdimport: no free carts available in specified group\n")); wavefile->closeWave(); import_failed_imports++; import_failed_imports++; @@ -1345,10 +1372,10 @@ MainObject::Result MainObject::ImportFile(const QString &filename, "`TITLE`='"+RDEscapeString(wavedata->title())+"'"; q=new RDSqlQuery(sql); if(q->first()) { - QString err_msg=QString(). - sprintf(" File \"%s\" has duplicate title \"%s\", skipping...\n", - RDGetBasePart(filename).toUtf8().constData(), - wavedata->title().toUtf8().constData()); + QString err_msg= + QString::asprintf(" File \"%s\" has duplicate title \"%s\", skipping...\n", + RDGetBasePart(filename).toUtf8().constData(), + wavedata->title().toUtf8().constData()); Log(LOG_WARNING,err_msg); import_journal->addFailure(effective_group->name(),filename, tr("duplicate title")); @@ -1406,11 +1433,11 @@ MainObject::Result MainObject::ImportFile(const QString &filename, settings->setAutotrimLevel(import_autotrim_level/100); conv->setDestinationSettings(settings); conv->setUseMetadata(import_update_metadata); - Log(LOG_INFO,QString(). - sprintf(" Importing file \"%s\" [%s] to cart %06u ... ", - RDGetBasePart(filename).toUtf8().constData(), - wavedata->title().trimmed().toUtf8().constData(), - *cartnum)); + Log(LOG_INFO, + QString::asprintf(" Importing file \"%s\" [%s] to cart %06u ... ", + RDGetBasePart(filename).toUtf8().constData(), + wavedata->title().trimmed().toUtf8().constData(), + *cartnum)); switch(conv_err=conv->runImport(rda->user()->name(),rda->user()->password(), &audio_conv_err)) { case RDAudioImport::ErrorOk: @@ -1690,10 +1717,15 @@ bool MainObject::OpenAudioFile(RDWaveFile **wavefile,RDWaveData *wavedata) void MainObject::VerifyFile(const QString &filename,unsigned *cartnum) { + LogDropBox("VerifyFile(\""+filename+"\")"); + bool found=false; QDateTime dt; - for(std::list::const_iterator + // + // Monitor Previously Detected Matches + // + for(std::list::const_iterator ci=import_dropbox_list.begin(); ci!=import_dropbox_list.end();ci++) { if((*ci)->filename==filename) { @@ -1704,10 +1736,12 @@ void MainObject::VerifyFile(const QString &filename,unsigned *cartnum) (file->lastModified().toSecsSinceEpoch()>dt.toSecsSinceEpoch())) { if((file->size()==(*ci)->size)&&(!(*ci)->failed)) { (*ci)->pass++; + LogDropBox("incremented pass count, "+(*ci)->dump()); } else { (*ci)->size=file->size(); (*ci)->pass=0; + LogDropBox("[1] size changed, reset pass, "+(*ci)->dump()); } if((*ci)->failed) { (*ci)->checked=true; @@ -1715,10 +1749,13 @@ void MainObject::VerifyFile(const QString &filename,unsigned *cartnum) (*ci)->failed=false; (*ci)->size=file->size(); (*ci)->pass=0; + LogDropBox("[2] size changed, reset pass, "+(*ci)->dump()); } } if((*ci)->pass>=RDIMPORT_DROPBOX_PASSES) { - switch(ImportFile(filename,cartnum)) { + MainObject::Result rslt=ImportFile(filename,cartnum); + LogDropBox(" Result: "+ResultText(rslt)); + switch(rslt) { case MainObject::Success: WriteTimestampCache(filename,file->lastModified()); break; @@ -1742,18 +1779,19 @@ void MainObject::VerifyFile(const QString &filename,unsigned *cartnum) (*ci)->checked=true; } } + else { + LogDropBox("stale file, ignoring"); + } delete file; } } + if(!found) { - QFile *file=new QFile(filename); - import_dropbox_list.push_back(new struct DropboxList()); - import_dropbox_list.back()->filename=filename; - import_dropbox_list.back()->size=file->size(); - import_dropbox_list.back()->pass=0; - import_dropbox_list.back()->checked=true; - import_dropbox_list.back()->failed=false; - delete file; + // + // No Remaining Matches, So Look For New Ones + // + import_dropbox_list.push_back(new DropboxList(filename)); + LogDropBox("adding new file, "+import_dropbox_list.back()->dump()); } } @@ -2440,7 +2478,7 @@ bool MainObject::LoadIsciXref(QString *err_msg,const QString &filename) } else { *err_msg=tr("invalid/corrupt data at line")+ - QString().sprintf("%d",3+fields.size()); + QString::asprintf("%d",3+fields.size()); return false; } } @@ -2552,6 +2590,15 @@ void MainObject::Log(int prio,const QString &msg) const } +void MainObject::LogDropBox(const QString &msg) const +{ + if(import_log_processing) { + Log(LOG_DEBUG, + QString::asprintf("[DB-%d]: ",import_persistent_dropbox_id)+msg+"\n"); + } +} + + void MainObject::NormalExit() const { if((import_journal!=NULL)&&(import_send_mail)) { @@ -2573,6 +2620,36 @@ void MainObject::ErrorExit(RDApplication::ExitCode code) const } +QString MainObject::ResultText(Result rslt) const +{ + QString ret=QString::asprintf("Unknown [%u]",rslt); + + switch(rslt) { + case MainObject::Success: + ret="Success"; + break; + + case MainObject::FileBad: + ret="FileBad"; + break; + + case MainObject::NoCart: + ret="NoCart"; + break; + + case MainObject::NoCut: + ret="NoCut"; + break; + + case MainObject::DuplicateTitle: + ret="DuplicateTitle"; + break; + } + + return ret; +} + + int main(int argc,char *argv[]) { QCoreApplication::setSetuidAllowed(true); diff --git a/utils/rdimport/rdimport.h b/utils/rdimport/rdimport.h index 00bfef28..3f481ee3 100644 --- a/utils/rdimport/rdimport.h +++ b/utils/rdimport/rdimport.h @@ -40,6 +40,7 @@ #include #include +#include "dropboxlist.h" #include "journal.h" #include "markerset.h" @@ -80,9 +81,11 @@ class MainObject : public QObject bool SchedulerCodeExists(const QString &code) const; void ReadXmlFile(const QString &basename,RDWaveData *wavedata) const; void Log(int prio,const QString &msg) const; + void LogDropBox(const QString &msg) const; void SendNotification(RDNotification::Action action,unsigned cartnum); void NormalExit() const; void ErrorExit(RDApplication::ExitCode code) const; + QString ResultText(Result rslt) const; unsigned import_file_key; RDGroup *import_group; bool import_verbose; @@ -147,14 +150,9 @@ class MainObject : public QObject QString import_string_user_defined; int import_string_year; int import_failed_imports; - struct DropboxList { - QString filename; - unsigned size; - unsigned pass; - bool checked; - bool failed; - }; + bool import_log_processing; std::list import_dropbox_list; + // QList import_dropbox_list; QString import_temp_fix_filename; MarkerSet *import_cut_markers; MarkerSet *import_talk_markers;