diff options
Diffstat (limited to 'kbabel/kbabeldict/modules/dbsearchengine')
19 files changed, 6274 insertions, 0 deletions
diff --git a/kbabel/kbabeldict/modules/dbsearchengine/AUTHOR b/kbabel/kbabeldict/modules/dbsearchengine/AUTHOR new file mode 100644 index 00000000..2a79312d --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/AUTHOR @@ -0,0 +1 @@ +Andrea Rizzi <rizzi@kde.org>
\ No newline at end of file diff --git a/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.cpp b/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.cpp new file mode 100644 index 00000000..ac55335d --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.cpp @@ -0,0 +1,1899 @@ +/*************************************************************************** + KDBSearchEngine.cpp - description + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + (C) 2005 by Stanislav Visnovsky + email : rizzi@kde.org + ***************************************************************************/ + +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + + License GPL v 2.0 + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * +*/ + +#include <qtextedit.h> +#include <qprogressdialog.h> + +#include <qinputdialog.h> +#include <kdeversion.h> +#include <klocale.h> +#include <kdebug.h> +#include <kio/netaccess.h> +#include <kmessagebox.h> +#include <kfiledialog.h> +#include <kurlrequester.h> +#include <kstandarddirs.h> + +#include "kapplication.h" +#include "KDBSearchEngine.h" + +#include "dbscan.h" + +#include "errno.h" +#include "stdio.h" +#include "stdlib.h" +#include <sys/time.h> +#include "preferenceswidget.h" +#include "dbse_factory.h" +#include <qprogressbar.h> +#include <qpushbutton.h> +#include <klineedit.h> +#include <kconfig.h> + +#include <qdir.h> +#include <qradiobutton.h> +#include <qcheckbox.h> +#include <qspinbox.h> +#include <qslider.h> +#include <qmemarray.h> + +#include "database.h" + +#include "catalogsettings.h" + +#define BIGNUMBER 400000000 + +using namespace KBabel; + +KDBSearchEngine::KDBSearchEngine (QObject * parent, const char *name): +SearchEngine (parent, name) +{ + edited = "unknown"; + dm = 0; //Database Manager + pw = 0; //Preference widget + lang = ""; + dbOpened = false; + dbname = ""; + lasterror = i18n ("No error"); + connect (this, SIGNAL (hasError (const QString &)), + SLOT (setLastError (const QString &))); + + IAmReady = true; // I don't know if it is a good idea, no DB loaded!!! + + scanInProgress = false; + searching = false; + stopNow = false; + + norm = false; // Normalize white space = FALSE + comm = true; // Remove Comments = TRUE + +} + + +KDBSearchEngine::~KDBSearchEngine () +{ +} + + + +bool +KDBSearchEngine::loadDatabase (QString database, bool noask = false) +{ + bool ret = true; + bool asked = false; + if (noask) + asked = true; + if (dm != 0) + { + delete dm; + dm = 0; + } + + QDir dir (database); + if (!dir.exists ()) + { + if (asked + || KMessageBox::questionYesNo (0, + i18n + ("Database folder does not exist:\n%1\n" + "Do you want to create it now?"). + arg (database), QString::null, i18n("Create Folder"), i18n("Do Not Create")) == + KMessageBox::Yes) + { + asked = true; + + QStringList dirList; + while (!dir.exists () && !dir.dirName ().isEmpty ()) + { + dirList.prepend (dir.dirName ()); + dir.setPath (dir.path () + "/.."); + } + + for (QStringList::Iterator it = dirList.begin (); + it != dirList.end (); ++it) + { + if (!dir.mkdir (*it)) + { + KMessageBox::sorry (0, + i18n + ("It was not possible to create folder %1"). + arg (dir.path () + "/" + + (*it))); + ret = false; + break; + } + dir.cd (*it); + } + } + else + { + ret = false; + } + } + + if (ret) + { + // test, if there are both of ,old and standard databases + QString transFile = database + "/translations." + lang + ".db"; + + bool oldstuff = QFile::exists (transFile + ",old"); + bool newstuff = QFile::exists (transFile); + + if (oldstuff && newstuff) + { + // there is an old db2 database, ask user + + if (KMessageBox:: + questionYesNo (0, + i18n + ("<p>There are backup database files from previous versions " + "of KBabel. However, another version of KBabel (probably from KDE 3.1.1 or 3.1.2) " + "created a new database. As a result, your KBabel installation contains two versions " + "of database files. Unfortunatelly, the old and new version " + "can not be merged. You need to choose one of them.<br/><br/>" + "If you choose the old version, the new one will be removed. " + "If you choose the new version, the old database files will be left alone " + "and you need to remove them manually. Otherwise this message will be displayed " + "again (the old files are at $KDEHOME/share/apps/kbabeldict/dbsearchengine/*,old).</p>"), + i18n ("Old Database Found"), + i18n ("Use &Old Database"), + i18n ("Use &New Database")) == + KMessageBox::Yes) + { + // remove the new files + QFile::remove (transFile); + QFile::remove (database + "/wordsindex." + lang + + ".db"); + QFile::remove (database + "/keysindex." + lang + ".db"); + QFile::remove (database + "/catalogsinfo." + lang + + ".db"); + + // rename the old files + KIO::NetAccess::copy (KURL (transFile + ",old"), + KURL (transFile), 0); + KIO::NetAccess:: + copy (KURL + (database + "/wordsindex." + lang + + ".db,old"), + KURL (database + "/wordsindex." + lang + + ".db"), 0); + KIO::NetAccess:: + copy (KURL + (database + "/keysindex." + lang + ".db,old"), + KURL (database + "/keysindex." + lang + + ".db"), 0); + KIO::NetAccess:: + copy (KURL + (database + "/catalogsinfo." + lang + + ".db,old"), + KURL (database + "/catalogsinfo." + lang + + ".db"), 0); + + QFile::remove (transFile + ",old"); + QFile::remove (database + "/wordsindex." + lang + + ".db,old"); + QFile::remove (database + "/keysindex." + lang + + ".db,old"); + QFile::remove (database + "/catalogsinfo." + lang + + ".db,old"); + } + } + else if (oldstuff) + { + // rename the old files + KIO::NetAccess::copy (KURL (transFile + ",old"), + KURL (transFile), 0); + KIO::NetAccess:: + copy (KURL (database + "/wordsindex." + lang + ".db,old"), + KURL (database + "/wordsindex." + lang + ".db"), 0); + KIO::NetAccess:: + copy (KURL (database + "/keysindex." + lang + ".db,old"), + KURL (database + "/keysindex." + lang + ".db"), 0); + KIO::NetAccess:: + copy (KURL + (database + "/catalogsinfo." + lang + ".db,old"), + KURL (database + "/catalogsinfo." + lang + ".db"), 0); + + QFile::remove (transFile + ",old"); + QFile::remove (database + "/wordsindex." + lang + ".db,old"); + QFile::remove (database + "/keysindex." + lang + ".db,old"); + QFile::remove (database + "/catalogsinfo." + lang + + ".db,old"); + } + + dm = new DataBaseManager (database, lang, this, "Database manager"); + + if (!dm->isOk ()) + { + if (asked + || KMessageBox::questionYesNo (0, + i18n + ("Database files not found.\nDo you want to create them now?"), QString::null, i18n("Create"), i18n("Do Not Create")) + == KMessageBox::Yes) + { + //fprintf(stderr,"SI\n"); + ret = dm->createDataBase (database, lang); + } + else + ret = false; + } + else + ret = true; + } + +//Wrong errore hangdling + + if (ret) + totalRecord = dm->count (); + else + totalRecord = 0; + + return ret; +} + + +/* + Set if the research have to consider multiple spaces as a single one. + */ + +void +KDBSearchEngine::setNormalizeSpace (bool normalize) +{ + norm = normalize; +} + + +void +KDBSearchEngine::setRemoveInternalComment (bool internalcomment) +{ + comm = internalcomment; +} + +/* + Set if the research have to be Case Sensitive or not + */ + +void +KDBSearchEngine::setCaseSensitive (bool sensitive) +{ + sens = sensitive; +} + +/* + Set the a string containing all char that must be ignored + during the search. + */ + +void +KDBSearchEngine::setRemoveCharString (QString chartoremove) +{ + remchar = chartoremove; +} + +/* + Return true if there's a search in progress. + */ +bool +KDBSearchEngine::isSearching () const +{ + return searching; +} + + + + +/* + Add a search string in the list of the string to search for. + Returns the ID of the string in the list. + Returns -1 if there is a problem (may be search in progress) + */ + +int +KDBSearchEngine::addSearchString (QString searchString, int rule) +{ + if (searching || scanInProgress) + return -1; + SearchEntry e; + e.string = QString (searchString); + e.rules = rule; + searchStringList.append (e); + return searchStringList.count (); +} + + +/* + Start the research in the database of all the string in the list + + */ + +bool +KDBSearchEngine::startSearch (const QString & str, uint pluralForm, + const SearchFilter * filter) +{ + + if (autoUpdate) + { + updateSettings (); + } + + + int l1 = 0, l2 = 0; + if (defSub1) + l1 = defLimit1; + if (defSub2) + l2 = defLimit2; + + return startSingleSearch (str, l1, l2); + + +} + +bool +KDBSearchEngine::startSearchInTranslation (QString s) +{ + + if (autoUpdate) + { + updateSettings (); + } + + + int l1 = 0, l2 = 0; + if (defSub1) + l1 = defLimit1; + if (defSub2) + l2 = defLimit2; + + + return startSingleSearch (s, l1, l2, true); + +} + + +bool +KDBSearchEngine::openDb (bool noask = false) +{ + if (!dbOpened) + { + dbOpened = loadDatabase (dbname, noask); //Try first to open it now + if (!dbOpened) // Still not opened!! + { + hasError (i18n ("Cannot open the database")); + return false; + } + } + + return true; +} + + + + +bool +KDBSearchEngine::messagesForFilter (const SearchFilter * filter, + QValueList < SearchResult > &resultList, + QString & error) +{ + + int count = 0; + stopNow = false; // Remove dirty. + SearchResult m; + + if (searching) + { + error = i18n ("Another search has already been started"); + return false; + } + + if (scanInProgress) + { + error = + i18n ("Unable to search now: a PO file scan is in progress"); + return false; + } + + + if (!openDb ()) + { + error = i18n ("Unable to open the database"); + return false; + } + + if (totalRecord <= 0) + { + error = i18n ("Database empty"); + return false; + } + + QString package = filter->location (); + + int step = (totalRecord / 30) + 1; + int ntra, nref; + int req = dm->searchCatalogInfo (package); + if (req == -1) + { + error = i18n ("No entry for this package in the database."); + return false; + } + DataBaseItem item; + int i, h; + kdDebug (0) << "looking for catalog " << req << endl; + + progressStarts (i18n ("Searching for %1 in database").arg (package)); + + for (item = dm->firstItem (); !item.isNull (); item = dm->nextItem ()) + { + count++; + if (count % step == 0) + { + emit progress (100 * count / totalRecord); + kapp->processEvents (100); + } + if (stopNow) + { + stopNow = false; + searching = false; + emit finished (); + return true; // No error, stopped! + } + + ntra = item.numTra; + for (i = 0; i < ntra; i++) + { + nref = item.translations[i].numRef; + + for (h = 0; h < nref; h++) + { + if (item.translations[i].infoRef[h] == req) + { + m.found = item.key; + m.translation = item.translations[i].translation; + resultList.append (m); + } + } + } + + } + + return true; +} + +void +KDBSearchEngine::repeat () +{ + + int count = 0; + stopNow = false; // Remove dirty. + + if (searching) + { + return; + } + + if (scanInProgress) + { + return; + } + + + if (!openDb ()) + { + return; + } + + if (totalRecord <= 0) + { + return; + } + + int step = (totalRecord / 30) + 1; + int ntra, nref; + + DataBaseItem item; + int i, h, tot; + + int req = dm->searchCatalogInfo ("kdelibs.po"); + if (req == -1) + kdDebug (0) << "No kdelibs.po found!" << endl; + + + QProgressDialog *pd = + new QProgressDialog (i18n ("Looking for repetitions"), i18n ("Stop"), + 100); + + connect (this, SIGNAL (progress (int)), pd, SLOT (setProgress (int))); + connect (this, SIGNAL (finished ()), pd, SLOT (close ())); + connect (pd, SIGNAL (cancelled ()), this, SLOT (stopSearch ())); + + + QString txt = "// %1 repetitions, %2 translation(s)\ni18n(\"%3\");\n"; + QString id; + int min; + bool ok = false; + + min = + QInputDialog::getInteger (i18n ("Minimum Repetition"), + i18n + ("Insert the minimum number of repetitions for a string:"), + 2, 1, 999999, 1, &ok); + + if (!ok) + return; + + pd->show (); + + progressStarts (i18n ("Searching repeated string")); + + static QTextEdit *mle = new QTextEdit (); + mle->clear (); + + bool inlibs; + + for (item = dm->firstItem (); !item.isNull (); item = dm->nextItem ()) + { + count++; + if (count % step == 0) + { + emit progress (100 * count / totalRecord); + kapp->processEvents (100); + } + if (stopNow) + { + stopNow = false; + searching = false; + emit finished (); + return; // No error, stopped! + } + tot = 0; + inlibs = false; + ntra = item.numTra; + for (i = 0; i < ntra; i++) + { + nref = item.translations[i].numRef; + for (h = 0; h < nref; h++) + if (item.translations[i].infoRef[h] == req) + inlibs = true; + + tot += nref; + } + + if (tot >= min && !inlibs) + { + id = item.key; + id = id.replace ("\n", "\"\n\""); + mle->append (txt.arg (tot).arg (ntra).arg (id)); + + } + } + + + emit progress (100); + emit finished (); + mle->resize (400, 400); + mle->show (); + + delete pd; + return; +} + + + +bool +KDBSearchEngine::startSearchNow (int searchmode) +{ + if (searchmode == -1) + searchmode = mode; + int count = 0; + stopNow = false; // Remove dirty. + clearResults (); + + + if (searching) + { + hasError (i18n ("Another search has already been started")); + return false; + } + + if (scanInProgress) + { + hasError (i18n + ("Unable to search now: a PO file scan is in progress")); + return false; + } + + + if (!openDb ()) + return false; + + + if (totalRecord <= 0) + { + hasError (i18n ("Database empty")); + return false; + } + + + + searching = true; + + emit started (); + + bool allkey = (searchmode == MD_ALL_GOOD_KEYS); + + bool equal, contains, contained, regexp, intra; + + intra = searchmode & MD_IN_TRANSLATION; + + QString msgIdFound; + QString msgId; + QString msgStr; + //QString msgIdRequested; + SearchResult *aresult; + TranslationInfo *adescription; + SearchList searchList; + int i, len, files, h; + len = remchar.length (); + + int n, m; //,word; + QString *id; + + QString mainRequest = searchStringList[0].string; + + + SearchList::Iterator it, it1; + QString *idMod; + bool foundSomething = false; + + searchList = searchStringList; //Create a copy and modify it + if (!allkey) + { + for (it = searchList.begin (); it != searchList.end (); ++it) + { + idMod = &((*it).string); + int pos; + for (i = 0; i < len; i++) + { + while ((pos = idMod->find (remchar.at (i))) != -1) + idMod->remove (pos, 1); + } + + if (comm) + idMod->replace (QRegExp ("\\_\\:.*\\\\n"), ""); //Read it from catalog !!! (NOT ONLY HERE) + + + if (norm) + idMod->simplifyWhiteSpace (); + if (!sens) + *idMod = idMod->upper (); + } + + } + + timeval now; + gettimeofday (&now, NULL); + //fprintf(stderr,"\n%ld.%ld\n",now.tv_sec,now.tv_usec); + //double tim=1.0*now.tv_usec/1000000.0+now.tv_sec; + int pos; + + + + DataBaseItem item; + + + //Now we can browse the whole db or the "good keys" + QValueList < KeyAndScore > goodkeys; + int totalprogress; + + bool gk = (searchmode == MD_GOOD_KEYS) || allkey; + int k = 0; + + if (gk) + { + goodkeys = searchWords (mainRequest, thre, threorig, listmax); //FIX IT, mainReq? + if (stopNow) + { + stopNow = false; + searching = false; + emit finished (); + return false; + } + if (goodkeys.count () == 0) + gk = false; // if no good keys, use the whole database + } + + // prepare progress values + totalprogress = gk ? goodkeys.count () : totalRecord; + int step = (totalprogress / 30) + 1; + if( step > 100 ) + step = 100; + + emit progress (0); + kapp->processEvents (100); + if (stopNow) + { + stopNow = false; + searching = false; + emit finished (); + return true; // No error, stopped! + } + + + for (item = gk ? (dm->getItem (goodkeys[0])) : (dm->firstItem ()); + !item.isNull (); + item = gk ? (dm->getItem (goodkeys[++k])) : (dm->nextItem ())) + { + +// Emit progress, process event and check stop now + if (count % step == 0) + { + emit progress (100 * count / /*QMAX( */ + totalprogress /*,1) */ ); + kapp->processEvents (100); + + if (stopNow) + { + stopNow = false; + searching = false; + emit finished (); + return true; // No error, stopped! + } + } + + // fprintf(stderr,"%s\n",(const char *)item.key.utf8()); + msgIdFound = item.key; //Check if this is OK with UTF8 + +// searchmode && MD_IN_TRANSLATION) + + count++; + + + + msgId = msgIdFound; + + if (!allkey) + { + + //Remove character in list of character to be ignored + for (i = 0; i < len; i++) + while ((pos = msgId.find (remchar.at (i))) != -1) + msgId.remove (pos, 1); + + //Remove context information from id found + if (comm) + msgId.replace (QRegExp ("\\_\\:.*\\\\n"), ""); + + + if (norm) + msgId.simplifyWhiteSpace (); + if (!sens) + msgId = msgId.upper (); + } + + + + it = searchList.begin (); + idMod = &((*it).string); + bool foundExact = false; + + for (it1 = searchStringList.begin (); + it1 != searchStringList.end (); it1++) + { + + id = &((*it1).string); + uint nn = 0; + do + { + if (intra) + { + msgId = item.translations[nn].translation; + if (!allkey) + { + //Remove character in list of character to be ignored + for (i = 0; i < len; i++) + while ((pos = + msgId.find (remchar.at (i))) != + -1) + msgId.remove (pos, 1); + + //Remove context information from id found + if (comm) + msgId. + replace (QRegExp ("\\_\\:.*\\\\n"), + ""); + + + if (norm) + msgId.simplifyWhiteSpace (); + if (!sens) + msgId = msgId.upper (); + } + + + } + + + int rules = (*it).rules; + + if (rules & Equal) + equal = (*idMod == msgId); + else + equal = false; + + if (rules & Contains) + contains = idMod->contains (msgId); + else + contains = false; + + if (rules & Contained) + contained = msgId.contains (*idMod); + else + contained = false; + + if (!foundExact && (rules & RegExp)) + { + QRegExp reg (*idMod); + regexp = (reg.search (msgId) != -1); + } + else + regexp = false; + nn++; + } + while (intra && nn < item.numTra); + + if (equal || contains || contained || regexp || allkey) + { + + if (equal) + foundExact = true; + + m = item.numTra; //Translations found. + + for (n = 0; n < m; n++) + { + + foundSomething = true; + + + msgStr = item.translations[n].translation; + + files = item.translations[n].numRef; + + aresult = new SearchResult (); + + results.setAutoDelete (true); + if (!gk) + aresult->score = + score (mainRequest, msgIdFound); + else + aresult->score = goodkeys[k].score; + + if (intra) + aresult->score = + score (mainRequest, + item.translations[n].translation); + + + SearchResult *s = 0; + for (s = results.first (); s != 0; + s = results.next ()) + { + if (s->score > aresult->score) + { + results.insert (results.at (), + aresult); + break; + } + + } + if (s == 0) //no break or empty list + results.append (aresult); + + +/* if(*id==msgIdFound) //Put it first of the list + results.prepend(aresult); + else + results.append(aresult); +*/ + aresult->requested = *id; + aresult->found = msgIdFound; + aresult->translation = msgStr; + aresult->descriptions.setAutoDelete (true); + for (h = 0; h < files; h++) + { + + aresult->descriptions.append (adescription = + new + TranslationInfo + ()); + int rr = item.translations[n].infoRef[h]; + + InfoItem info = dm->getCatalogInfo (rr); + + + adescription->location = info.catalogName; + adescription->translator = + info.lastTranslator; + adescription->filePath = info.lastFullPath; + } + + emit numberOfResultsChanged (results.count ()); + emit resultFound (aresult); + +// if(*id==msgIdFound) //Put it first of the list so th order change + emit resultsReordered (); + + + + } + } + // idMod=searchList.next(); + it++; + idMod = &((*it).string); + } + + + } + gettimeofday (&now, NULL); + //fprintf(stderr,"%ld.%ld\n",now.tv_sec,now.tv_usec); + + //fprintf(stderr,"Finish, %d (of %d) records in %f seconds!!\n",count,totalRecord, 1.0*now.tv_usec/1000000.0+now.tv_sec-tim); + emit progress (100); + emit finished (); + + searching = false; + return true; //foundSomething; +} + +/* + Start a search for a single string + */ + +bool +KDBSearchEngine::startSingleSearch (QString searchString, + unsigned int pattern1Limit, + unsigned int /*pattern2Limit */ , + bool inTranslation) +{ + /* + Check Ret + value. + */ + + unsigned int nw = 0; + int in = 0, len = 0; + clearList (); + addSearchString (searchString, defRule); + + + + QRegExp reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); + while ((in = reg.search (searchString, in + len)) != -1) + { + nw++; + len = reg.matchedLength (); + } + in = 0; + len = 0; + // fprintf(stderr,"asas %d\n",nw); + + if (mode == MD_ALL_GOOD_KEYS && !inTranslation) + return startSearchNow (); + + + + if ((nw < pattern1Limit) && (nw > 1)) + for (unsigned int k = 0; k < nw; k++) + { + in = reg.search (searchString, in + len); + len = reg.matchedLength (); + QString regToAdd = searchString; + regToAdd.replace (in, len, "[a-zA-Z0-9_%" + regaddchar + "]*"); + regToAdd.append ("$"); + regToAdd.prepend ("^"); +// fprintf(stderr,"%s",(const char *)regToAdd.local8Bit()); + addSearchString (regToAdd, RegExp); + } + + if (inTranslation) + return startSearchNow (MD_IN_TRANSLATION); + else + return startSearchNow (); + + return false; + +} + + +/* + Start a search for a list of string + */ + +//bool KDBSearchEngine::startListSearch(QPtrList<QString> searchStrList) +//{ + // searchStringList=searchStrList; +// return startSearchNow(); +//} +/* + Stop the current search + */ + + + +void +KDBSearchEngine::setLanguageCode (const QString & ll) +{ + if (ll == lang) + return; + + lang = ll; + if (dbOpened) //if opened open it again with new code, what about close before open ? + dbOpened = loadDatabase (dbname); + +} + +void +KDBSearchEngine::setLanguage (const QString & languageCode, + const QString & /*languageName */ ) +{ + setLanguageCode (languageCode); +} + +void +KDBSearchEngine::stopSearch () +{ + stopNow = true; +} + + +void +KDBSearchEngine::clearList () +{ + searchStringList.clear (); +} + +bool +KDBSearchEngine::isReady () const +{ + return IAmReady; +} + + +void +KDBSearchEngine::saveSettings (KConfigBase * config) +{ +// updateSettings(); //maybe with autoupdate + KConfigGroupSaver cgs (config, "KDBSearchEngine"); +#if KDE_IS_VERSION(3,1,3) + config->writePathEntry ("Filename", dbname); +#else + config->writeEntry ("Filename", dbname); +#endif + config->writeEntry ("Language", lang); + + + config->writeEntry ("CaseSensitive", sens); + config->writeEntry ("Normalize", norm); + config->writeEntry ("RemoveContext", comm); + + config->writeEntry ("Rules", defRule); + config->writeEntry ("Limit1", defLimit1); + config->writeEntry ("Limit2", defLimit2); + config->writeEntry ("Substitution1", defSub1); + config->writeEntry ("Substitution2", defSub2); + + config->writeEntry ("RegExp", regaddchar); + config->writeEntry ("RemoveCharacter", remchar); + + config->writeEntry ("Threshold1", thre); + config->writeEntry ("Threshold2", threorig); + config->writeEntry ("ListMax", listmax); + config->writeEntry ("Mode", mode); + config->writeEntry ("CommonThrs", commonthre); + config->writeEntry ("ReturnNothing", retnot); + + config->writeEntry ("AutoAuthor", autoauthor); + config->writeEntry ("AutoUp", autoup); + +} + + +void +KDBSearchEngine::readSettings (KConfigBase * config) +{ + QString newName; + + KConfigGroupSaver cgs (config, "KDBSearchEngine"); + + QString defaultLang; + QString oldLang = lang; + Defaults::Identity def; + defaultLang = def.languageCode (); + lang = config->readEntry ("Language", defaultLang); + + QString defaultDir; + KStandardDirs *dirs = KGlobal::dirs (); + if (dirs) + { + defaultDir = dirs->saveLocation ("data"); + if (defaultDir.right (1) != "/") + defaultDir += "/"; + defaultDir += "kbabeldict/dbsearchengine"; + } + + newName = config->readPathEntry ("Filename", defaultDir); + + if (newName != dbname || oldLang != lang) + { + dbname = newName; + if (dbOpened) //Reload only if it is opened + dbOpened = loadDatabase (dbname); + } + + sens = config->readBoolEntry ("CaseSensitive", false); + norm = config->readBoolEntry ("Normalize", true); + comm = config->readBoolEntry ("RemoveContext", true); + + defRule = config->readNumEntry ("Rules", 1); + defLimit1 = config->readNumEntry ("Limit1", 20); + defLimit2 = config->readNumEntry ("Limit2", 8); + + thre = config->readNumEntry ("Threshold1", 50); + threorig = config->readNumEntry ("Threshold2", 50); + listmax = config->readNumEntry ("ListMax", 500); + mode = config->readNumEntry ("Mode", MD_GOOD_KEYS); + + defSub1 = config->readBoolEntry ("Substitution1", true); + defSub2 = config->readBoolEntry ("Substitution2", false); + + regaddchar = config->readEntry ("RegExp"); + remchar = config->readEntry ("RemoveCharacter", "&.:"); + commonthre = config->readNumEntry ("CommonThrs", 300); + retnot = config->readBoolEntry ("ReturnNothing", false); + autoauthor = config->readEntry ("AutoAuthor"); + autoup = config->readBoolEntry ("AutoUp", true); + + setSettings (); +} + +PrefWidget * +KDBSearchEngine::preferencesWidget (QWidget * parent) +{ + + pw = new PreferencesWidget (parent); + setSettings (); + connect (pw, SIGNAL (restoreNow ()), this, SLOT (setSettings ())); + connect (pw, SIGNAL (applyNow ()), this, SLOT (updateSettings ())); + connect (pw, SIGNAL (destroyed ()), this, SLOT (prefDestr ())); + connect (pw->dbpw->scanPB_2, SIGNAL (clicked ()), this, SLOT (scan ())); + connect (pw->dbpw->scanrecPB, SIGNAL (clicked ()), this, + SLOT (scanRecur ())); + connect (pw->dbpw->scanFilePB, SIGNAL (clicked ()), this, + SLOT (scanFile ())); + connect (pw->dbpw->repeatPB, SIGNAL (clicked ()), this, SLOT (repeat ())); + + + return pw; +} + +void +KDBSearchEngine::scanRecur () +{ + if (scanInProgress) + return; + updateSettings (); + + if (!openDb ()) + return; + scanInProgress = true; + PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); + QString cvsdir; + cvsdir = + KFileDialog::getExistingDirectory ("", 0, + i18n + ("Select Folder to Scan Recursively")); + + if (cvsdir.isEmpty ()) + { + scanInProgress = false; + return; + } + if (pw) + { + connect (sca, SIGNAL (patternProgress (int)), pw->dbpw->totalPB, + SLOT (setProgress (int))); + connect (sca, SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, + SLOT (setProgress (int))); + connect (sca, SIGNAL (fileProgress (int)), pw->dbpw->processPB, + SLOT (setProgress (int))); + } + + connect (sca, SIGNAL (patternProgress (int)), SIGNAL (progress (int))); //Kbabel progress bar + + connect (sca, SIGNAL (added (int)), pw, SLOT (setEntries (int))); + connect (sca, SIGNAL (filename (QString)), pw, SLOT (setName (QString))); + + + progressStarts (i18n ("Scanning folder %1").arg (cvsdir)); + connect (sca, SIGNAL (patternFinished ()), SIGNAL (progressEnds ())); + + sca->scanPattern (cvsdir, "*.po", true); + disconnect (this, SIGNAL (progress (int))); +//disconnect(SIGNAL(patternStarted()),this,SIGNAL(started()) ); + disconnect (this, SIGNAL (progressEnds ())); + if (pw) + { + disconnect (pw->dbpw->totalPB, SLOT (setProgress (int))); + disconnect (pw->dbpw->loadingPB, SLOT (setProgress (int))); + disconnect (pw->dbpw->processPB, SLOT (setProgress (int))); + } + totalRecord = dm->count (); + + scanInProgress = false; + dm->sync (); + delete sca; +} + + +void +KDBSearchEngine::scan () +{ + if (scanInProgress) + return; + updateSettings (); + + if (!openDb ()) + return; + scanInProgress = true; + PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); + QString cvsdir; + + cvsdir = + KFileDialog::getExistingDirectory ("", 0, + i18n ("Select Folder to Scan")); + if (cvsdir.isEmpty ()) + { + scanInProgress = false; + return; + } + if (pw) + { + connect (sca, SIGNAL (patternProgress (int)), pw->dbpw->totalPB, + SLOT (setProgress (int))); + connect (sca, SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, + SLOT (setProgress (int))); + connect (sca, SIGNAL (fileProgress (int)), pw->dbpw->processPB, + SLOT (setProgress (int))); + } + connect (sca, SIGNAL (patternProgress (int)), SIGNAL (progress (int))); + progressStarts (i18n ("Scanning folder %1").arg (cvsdir)); + connect (sca, SIGNAL (patternFinished ()), SIGNAL (progressEnds ())); + + connect (sca, SIGNAL (added (int)), pw, SLOT (setEntries (int))); + connect (sca, SIGNAL (filename (QString)), pw, SLOT (setName (QString))); + + + + sca->scanPattern (cvsdir, "*.po", false); + + disconnect (this, SIGNAL (progress (int))); +//disconnect(SIGNAL(patternStarted()),this,SIGNAL(started()) ); + disconnect (this, SIGNAL (progressEnds ())); + if (pw) + { + disconnect (pw->dbpw->totalPB, SLOT (setProgress (int))); + disconnect (pw->dbpw->loadingPB, SLOT (setProgress (int))); + disconnect (pw->dbpw->processPB, SLOT (setProgress (int))); + } + totalRecord = dm->count (); + + scanInProgress = false; + + dm->sync (); + delete sca; +} + +void +KDBSearchEngine::scanFile () +{ + if (scanInProgress) + return; + updateSettings (); + + if (!openDb ()) + return; + scanInProgress = true; + PoScanner *sca = new PoScanner (dm, this, "Po Scanner"); + QString cvsdir; + pw->dbpw->totalPB->setProgress (0); + + cvsdir = + KFileDialog::getOpenFileName ("", "*.po", 0, + i18n ("Select PO File to Scan")); + if (cvsdir.isEmpty ()) + { + scanInProgress = false; + return; + } + if (pw) + { + connect (sca, SIGNAL (fileLoading (int)), pw->dbpw->loadingPB, + SLOT (setProgress (int))); + connect (sca, SIGNAL (fileProgress (int)), pw->dbpw->processPB, + SLOT (setProgress (int))); + } + connect (sca, SIGNAL (fileProgress (int)), SIGNAL (progress (int))); + progressStarts (i18n ("Scanning file %1").arg (directory (cvsdir, 0))); + connect (sca, SIGNAL (fileFinished ()), SIGNAL (progressEnds ())); + + connect (sca, SIGNAL (added (int)), pw, SLOT (setEntries (int))); + connect (sca, SIGNAL (filename (QString)), pw, SLOT (setName (QString))); + + + + sca->scanFile (cvsdir); + + sca->disconnect (SIGNAL (fileProgress (int)), this, + SIGNAL (progress (int))); +//disconnect(SIGNAL(patternStarted()),this,SIGNAL(started()) ); + sca->disconnect (SIGNAL (fileFinished ()), this, + SIGNAL (progressEnds ())); + if (pw) + { + disconnect (pw->dbpw->loadingPB, SLOT (setProgress (int))); + disconnect (pw->dbpw->processPB, SLOT (setProgress (int))); + } + + totalRecord = dm->count (); + + scanInProgress = false; + + dm->sync (); + delete sca; +} + +const KAboutData * +KDBSearchEngine::about () const +{ + + return DbSeFactory::instance ()->aboutData (); + +} + +QString +KDBSearchEngine::name () const +{ + return i18n ("Translation Database"); +} + +QString +KDBSearchEngine::id () const +{ + return QString ("KDBSearchEngine"); +} + +QString +KDBSearchEngine::lastError () +{ + return lasterror; +} + +void +KDBSearchEngine::prefDestr () +{ + pw = 0; +} + + +void +KDBSearchEngine::setSettings () +{ + + if (pw == 0) + return; + + pw->dbpw->dirInput->setURL (dbname); + pw->dbpw->caseSensitiveCB->setChecked (sens); + pw->dbpw->normalizeCB->setChecked (norm); + pw->dbpw->removeContextCB->setChecked (comm); + + pw->dbpw->oneWordSubCB->setChecked (defSub1); + pw->dbpw->twoWordSubCB->setChecked (defSub2); + + + if (defRule == 8) + pw->dbpw->RegExpRB->setChecked (true); + else + { + pw->dbpw->normalTextRB->setChecked (true); + pw->dbpw->equalCB->setChecked (defRule & Equal); + pw->dbpw->containsCB->setChecked (defRule & Contains); + pw->dbpw->containedCB->setChecked (defRule & Contained); + } + + pw->dbpw->oneWordSubSB->setValue (defLimit1); + pw->dbpw->twoWordSubSB->setValue (defLimit2); + + pw->dbpw->maxSB->setValue (listmax); + pw->dbpw->thresholdSL->setValue (thre); + pw->dbpw->thresholdOrigSL->setValue (threorig); + + pw->dbpw->allRB->setChecked (mode == MD_ALL_DB); + pw->dbpw->slistRB->setChecked (mode == MD_GOOD_KEYS); + pw->dbpw->rlistRB->setChecked (mode == MD_ALL_GOOD_KEYS); + + pw->dbpw->nothingCB->setChecked (retnot); + pw->dbpw->freqSB->setValue (commonthre); + + pw->dbpw->regExpLE->setText (regaddchar); + pw->dbpw->ignoreLE->setText (remchar); + + pw->dbpw->authorLE->setText (autoauthor); + pw->dbpw->autoAddCB_2->setChecked (autoup); + + + +} + + +void +KDBSearchEngine::updateSettings () +{ + if (pw == 0) + return; + + QString newName = pw->dbpw->dirInput->url (); + + if (newName != dbname) + { + kdDebug (0) << "Database changed" << endl; + dbname = newName; + if (dbOpened) + dbOpened = loadDatabase (dbname); + } + + sens = pw->dbpw->caseSensitiveCB->isChecked (); + norm = pw->dbpw->normalizeCB->isChecked (); + comm = pw->dbpw->removeContextCB->isChecked (); + + int tmpRule = 0; + if (pw->dbpw->RegExpRB->isChecked ()) + tmpRule = RegExp; + else + { + if (pw->dbpw->equalCB->isChecked ()) + tmpRule += Equal; + if (pw->dbpw->containsCB->isChecked ()) + tmpRule += Contains; + if (pw->dbpw->containedCB->isChecked ()) + tmpRule += Contained; + } + + defRule = tmpRule; + + defLimit1 = pw->dbpw->oneWordSubSB->text ().toInt (); + defLimit2 = pw->dbpw->twoWordSubSB->text ().toInt (); + defSub1 = pw->dbpw->oneWordSubCB->isChecked (); + defSub2 = pw->dbpw->twoWordSubCB->isChecked (); + + listmax = pw->dbpw->maxSB->value (); + thre = pw->dbpw->thresholdSL->value (); + threorig = pw->dbpw->thresholdOrigSL->value (); + + if (pw->dbpw->allRB->isChecked ()) + mode = MD_ALL_DB; + if (pw->dbpw->slistRB->isChecked ()) + mode = MD_GOOD_KEYS; + if (pw->dbpw->rlistRB->isChecked ()) + mode = MD_ALL_GOOD_KEYS; + + + regaddchar = pw->dbpw->regExpLE->text (); + remchar = pw->dbpw->ignoreLE->text (); + + retnot = pw->dbpw->nothingCB->isChecked (); + commonthre = pw->dbpw->freqSB->value (); + + autoauthor = pw->dbpw->authorLE->text (); + autoup = pw->dbpw->autoAddCB_2->isChecked (); + + +} + +void +KDBSearchEngine::setLastError (const QString & er) +{ + lasterror = er; +} + +QString +KDBSearchEngine::translate (const QString & text, const uint pluralForm) +{ + if (!openDb ()) + return QString::null; +/* + +if(!dbOpened) + { + dbOpened=loadDatabase(dbname); //Try first to open it now + if(!dbOpened) // Still not opened!! + { + //emit anerror + hasError(i18n("Database not opened")); + return QString::null; + } + } +*/ + + DataBaseItem dbit = dm->getItem (text); + + if (dbit.isNull ()) + return QString::null; + if (dbit.numTra == 1) + return dbit.translations[0].translation; + + uint32 n = dbit.numTra; + uint32 max = 0, nmax = 0; + for (uint32 i = 0; i < n; i++) + if (dbit.translations[i].numRef > max) + { + nmax = i; + max = dbit.translations[i].numRef; + } + + return dbit.translations[nmax].translation; + + +} + +QValueList < KeyAndScore > KDBSearchEngine::searchWords (QString phrase, + int threshold, + int thresholdorig, + uint32 max) +{ + QValueList < QString > wordlist; + + if (!openDb ()) + { + QValueList < KeyAndScore > a; + return a; + } + + progressStarts (i18n ("Searching words")); + + QValueList < QString >::Iterator wlit; + wordlist = dm->wordsIn (phrase); + int + nw = wordlist.count (); +//QMemArray<WordItem> wi(nw); + QMemArray < uint32 > numofloc (nw), currentloc (nw); + QMemArray < int > + score (nw); + QMemArray < uint32 * >loc (nw), locorig (nw); + QValueList < uint32 > resloc; + QValueList < int > + resfound; + QValueList < KeyAndScore > keylist; +//wi.resize(wordlist.count()); + int + totalprogress = 0; + int + totrec = dm->count (); + uint32 + cthre = totrec * commonthre / 10000; + int + i = 0, common = 0; + for (wlit = wordlist.begin (); wlit != wordlist.end (); ++wlit) + { + WordItem + wi = dm->getWordLocations (*wlit); + if (!wi.notFound ()) + { + if (wi.count < cthre) + score[i] = 1; + else + { + score[i] = 0; + common++; + } + + locorig[i] = loc[i] = wi.locations; + totalprogress += numofloc[i] = wi.count; + currentloc[i] = 0; +// score[i]=wi.score; + //wi[i]=WordItem(wi[i]); + //wi[i].locations.detach(); + i++; +// } + // else + // common++; + + + + } + } + bool + cs = (common == nw); //All words are common; + if (totalprogress == 0) + totalprogress = 1; + int + step = totalprogress / 30 + 1; + int + count = 0; + int + thrs = (wordlist.count () * threshold) / 100; + if (thrs < 1) + thrs = 1; // whole database ??? + + int + tot = i; +//nt32 jmin=0; + int + found; + uint32 + min; //Big ? + bool + empty = false; + + + while (!empty) + { + + empty = true; + found = retnot ? common : 0; + if (thrs <= found) + thrs = found + 1; // whole database ??? + + min = BIGNUMBER; + for (int j = 0; j < tot; j++) + if (cs || score[j]) + { + if (numofloc[j] > currentloc[j]) // Check if there's still something to do. + empty = false; + if (loc[j][0] < min) //Found the minimum head + min = loc[j][0]; + + + } + if (min != BIGNUMBER) + { + for (int j = 0; j < tot; j++) + if (cs || score[j]) + { + if (loc[j][0] == min) //Count the heads, move forward + { + found++; + count++; + //check stopnow here + if (count % step == 0) + { + emit + progress (100 * count / totalprogress); + kapp->processEvents (100); + } + if (stopNow) + { + return keylist; + } + + currentloc[j]++; + if (numofloc[j] == currentloc[j]) //End reached + loc[j][0] = BIGNUMBER; //so set head to a big number + else //Go on.. + { + loc[j]++; + } + } + } //end of for + bool + inserted = false; + + + + if (found >= thrs) + { + //Words count in key. + int + nword = 0; + + int + in = 0, len = 0; + QString + keyst = dm->getKey (min); + QRegExp + reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); + while ((in = reg.search (keyst, in + len)) != -1) + { + nword++; + len = reg.matchedLength (); + } + + if (found >= nword * thresholdorig / 100) // + { + + if (resfound.count () <= max + || (*resfound.end () < found)) + if ((*resfound.end ()) >= found) + { + inserted = true; + resloc.append (min); + resfound.append (found); + + } + else + for (uint32 j = 0; j < resloc.count (); + j++) + { + if (resfound[j] < found || (resfound[j] == found && 0)) //Orig word + { + resloc.insert (resloc.at (j), + min); + resfound.insert (resfound. + at (j), + found); + inserted = true; + break; + } + } + if (!inserted) + { + resloc.append (min); + resfound.append (found); + } + + } + } + + } + + } + int + nres = (resloc.count () < max) ? resloc.count () : max; + + for (int j = 0; j < nres; j++) + { + QString + strkey = dm->getKey (resloc[j]); + int + stdscore = KDBSearchEngine::score (phrase, strkey); + int + sc = 0; + + if (stdscore < 99) + { + int + in = 0, len = 0, nword = 0; + int + remove = retnot ? common : 0; + QRegExp + reg ("[a-zA-Z0-9_%" /*+remchar */ + regaddchar + "]+"); + while ((in = reg.search (strkey, in + len)) != -1) + { + nword++; + len = reg.matchedLength (); + } + +// kdDebug(0) << nword << "NWORD " << resfound[j] << "FOUND " +// << resfound[j]-remove << "REAL " << remove << "to be remove " << endl; + if (nword < 1) + nword = 1; + sc = 70 - resfound[j] * 70 / nw + abs (nword - + (resfound[j] - + remove)) * 30 / + nword + 2; + sc = 100 - sc; +// kdDebug(0) <<" Score : " << sc << endl; + + } + else + sc = stdscore; + + KeyAndScore + key (strkey, sc); + +// kdDebug(0) << (QString) key << " [" << key.score << "]" << endl; + keylist.append (key); + } + +//kdDebug(0) << "Here!" << endl; + + for (int j = 0; j < tot; j++) + { + free (locorig[j]); + } + progressStarts (i18n ("Process output")); + return keylist; +} + +void +KDBSearchEngine::stringChanged (const QStringList & o, + const QString & translated, const uint, + const QString &) +{ + + QString orig = o.first (); // FIXME: plural forms + + // skip empty originals or translated texts + if (orig.isEmpty () || translated.isEmpty ()) + return; + + if (autoup) + { + if (openDb (true)) //true= no ask + { + + dm->putNewTranslation (orig, translated, + dm->catalogRef (directory (edited, 0), + autoauthor, edited)); + //kdDebug(0) << "Changed " << orig << " " << translated << endl; + dm->sync (); + } + } + +} + +void +KDBSearchEngine::setEditedFile (const QString & file) +{ + + edited = file; //kdDebug(0) << edited << endl; +} + +KeyAndScore::KeyAndScore (const QString & a, int sc): +QString (a) +{ + score = sc; +} + +KeyAndScore::KeyAndScore ():QString () +{ + score = 0; +} + +#include "KDBSearchEngine.moc" diff --git a/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.h b/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.h new file mode 100644 index 00000000..058444db --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/KDBSearchEngine.h @@ -0,0 +1,333 @@ + +/*************************************************************************** + KDBSearchEngine.h - description + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + email : rizzi@kde.org + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; version 2 of the License. * + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * + ***************************************************************************/ +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + +*/ + + +/* + * This is a database engine specific + * structure, I think I must found a way to + * make it more "standard" + */ + +#ifndef _KDBSEARCH_ENGINE_ +#define _KDBSEARCH_ENGINE_ +#include <qobject.h> +#include <qptrlist.h> +#include <qstring.h> +#include <qmemarray.h> +#include <qvaluelist.h> + + +#include "database.h" +#include "searchengine.h" +#include "preferenceswidget.h" + + +//value of mode +#define MD_ALL_DB 1 +#define MD_GOOD_KEYS 2 +#define MD_ALL_GOOD_KEYS 3 +#define MD_IN_TRANSLATION 4 + +class SearchEntry +{ + public: + QString string; + int rules; +}; + +class KeyAndScore : public QString +{ + public: + KeyAndScore(); + KeyAndScore(const QString &a,int sc); + int score; +}; + +typedef QValueList<SearchEntry> SearchList; + +/* USE searchengine.h structure +class CatalogDescription +{ + public: + char language[6]; // How many character chinese language needs? + // QString is better ? + QString filename; + QString dateOfScan; //When you add this to database (last time) + QString authorOfScan; + QString fileHeader; //msgid "" +}; + + */ + +class KDBSearchEngine : public SearchEngine +{ + Q_OBJECT + +public: + + + KDBSearchEngine(QObject *parent=0,const char *name=0); + ~KDBSearchEngine(); + enum Rules {Equal = 1, Contains = 2, Contained = 4, RegExp = 8}; + enum FormatRules { Dots = 1, Ampersand = 2, FirstWordUpper = 4, //Not yet used + AllWordUpper = 8, AmpersanAlwaysFirst = 16 }; + + /* + Set the directory where db file are located + */ + + void setDBName(QString filename); + + /* + Set if the research have to consider multiple spaces as a single one. + */ + + void setNormalizeSpace(bool normalize); + + /* + Set if the research have to be Case Sensitive or not + */ + + void setCaseSensitive(bool sensitive); + + + /* + ignore ":_ bla bla \n" in msgid + + */ + + void setRemoveInternalComment(bool internalcomment); + + /* + Set the a string containing all char that must be ignored + during the search. + */ + + + void setRemoveCharString(QString chartoremove); + + /* + Enable an output filter that format Uppercase/lowercase + method could be; + 0 = Disabled; + 1 = English pattern; + 2 = Only first; + */ + + void enableCapitalFilter(int method); + + /* + Enable an output filter that format special characters + method could be; + 0 = Disabled; + 1 = Put ... at the end if needed (remove if present but no needed); + 2 = Put an accelerator (or remove); + 4 = Remove %* if not present in msgid. + 8 = Change %d,%s etc with %1 %2 etc if needed. + */ + + void enableCharFilter(int method); + + /* + Sets the rules to decide when 2 string match (Uppercase and remove char + are not set here) + 1 = EQUAL + 2 = SEARCH STRING IS CONTAINED IN DATABASE STRING (use with care, if you search nothing + it will produce a dangerouse output) + 4 = DATABASE STRING IS CONTAINED IN SEARCH STRING (it exclude msgid "") + 8 = String is a regexp. (exclude 1 2 4) + */ + + void setMatchingRules(int rules); + + /* + Search Engine methods reimplemented + */ + + + + virtual bool messagesForFilter(const SearchFilter* filter + , QValueList<SearchResult>& resultList, QString& error); + + + virtual void setLanguageCode(const QString& lang); + virtual void setLanguage(const QString& languageCode, const QString& languageName); + + virtual void setEditedFile(const QString& file); + + virtual bool isReady() const ; + virtual bool isSearching() const; + + virtual void saveSettings(KConfigBase *config); + virtual void readSettings(KConfigBase *config); + + virtual PrefWidget *preferencesWidget(QWidget *parent); + + virtual const KAboutData *about() const; + + virtual QString name() const; + + virtual QString id() const; + + virtual QString lastError(); + + virtual QString translate(const QString& text, uint pluralForm); + + virtual void stringChanged( const QStringList& orig, const QString& translated + , const uint translationPluralForm, const QString& description); + + +public slots: + + void scan(); + void scanRecur(); + void scanFile(); + /* + Add a search string in the list of the string to search for. + Returns the ID of the string in the list. + Returns -1 if there is a problem (may be search in + progress) + */ + + int addSearchString(QString searchString, int rule=Equal); + + /* + Start the research in the database of all the string in the list + search mode -1 means use global mode; + */ + bool startSearchNow(int searchmode=-1); + + /* + Start a search for a single string + If the number of word of your string is less than patternXlimit + the KDBSearchEngine will search for all string with X * substituted to + X word. + For example if pattern1limit is 4 and you search "My name is Andrea" + This string will also match with: + "Your name is Andrea", "My dog is Andrea", "My name was Andrea", "My name is Joe" + + Do not set pattern2limit too high. + + If a patternlimit is > 0 rules 8 (* means any word) is added. + + */ + + bool startSingleSearch(QString searchString,unsigned int pattern1Limit,unsigned int pattern2Limit, + bool inTranslation=false); + + /* + Start a search for a list of string + */ + +// bool startListSearch(QPtrList<QString> searchStrList); + +/* + * Return a list of key in database that contains some + * words of the given string phrase. + * thresholdin is a number from 0 to 100 a key is good + * if at least the threshold% of words are found + * thresholdout a key is good if the found words represnt + * at least the thresholdorig% of the words in the key + * max is the maximum number of results + */ + +QValueList<KeyAndScore> searchWords(QString phrase,int threshold, + int thresholdorig ,uint32 max); + + + /* + Stop the current search + */ + + virtual void stopSearch(); + virtual bool startSearch(const QString& text, uint pluralForm, const SearchFilter* filter); + virtual bool startSearchInTranslation(QString s); + + void clearList(); +signals: + void found(SearchResult *); +private slots: + void updateSettings(); //Use widget settings + void setSettings(); //Fill widget with actual settings + void prefDestr(); + void setLastError(const QString&); + void repeat(); +private: + /* + Reload database info (and keep the dbase opened). + */ + bool loadDatabase(QString database,bool); + + bool openDb(bool); + + PreferencesWidget *pw; + bool IAmReady; + int methodFilterChar; + int methodFilterCapital; + int defRule; + int defLimit1; + int defLimit2; + int thre; + int threorig; + int commonthre; + int listmax; + int mode; + bool retnot; + bool defSub1; + bool defSub2; + bool stopNow; + bool searching; + bool norm,sens,comm; + int numofresult; + char * filename; + QString remchar; + QString regaddchar; + QString dbname; + bool dbOpened; +// GDBM_FILE db; + DataBaseManager * dm; +// datum key, value; +// QMemArray<CatalogDescription *> catinfo; + SearchList searchStringList; + int totalRecord; + QString lasterror; + QString lang; + bool scanInProgress; + QString edited; + bool autoup; + QString autoauthor; +}; + +#endif diff --git a/kbabel/kbabeldict/modules/dbsearchengine/Makefile.am b/kbabel/kbabeldict/modules/dbsearchengine/Makefile.am new file mode 100644 index 00000000..8504999a --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/Makefile.am @@ -0,0 +1,34 @@ +## Makefile.am for DBSE + +# this has all of the subdirectories that make will recurse into. if +# there are none, comment this out +#SUBDIRS = + + +# this is the program that gets installed. it's name is used for all +# of the other Makefile.am variables +kde_module_LTLIBRARIES = kbabeldict_dbsearchengine.la + +# set the include path for X, qt and KDE +INCLUDES = -I$(srcdir)/../.. -I../../../common -I$(srcdir)/../../../common $(all_includes) $(DBIV_INCLUDES) + +# which sources should be compiled for kbabel +kbabeldict_dbsearchengine_la_SOURCES = KDBSearchEngine.cpp preferenceswidget.cpp \ + dbse_factory.cpp dbseprefwidget.ui database.cpp dbscan.cpp +#database.cpp dbscan.cpp +kbabeldict_dbsearchengine_la_LIBADD = ../../libkbabeldictplugin.la ../../../common/libkbabelcommon.la $(LIB_KDEUI) $(LIB_KIO) $(LIB_DBIV) +kbabeldict_dbsearchengine_la_LDFLAGS = $(all_libraries) -module -avoid-version -no-undefined \ + $(DBIV_LDFLAGS) + + +# these are the headers for your project +noinst_HEADERS = KDBSearchEngine.h preferenceswidget.h dbse_factory.h \ + database.h dbscan.h + + +# let automoc handle all of the meta source files (moc) +METASOURCES = AUTO + +kde_services_DATA = dbsearchengine.desktop +EXTRA_DIST = $(kde_services_DATA) + diff --git a/kbabel/kbabeldict/modules/dbsearchengine/STRUCTURE b/kbabel/kbabeldict/modules/dbsearchengine/STRUCTURE new file mode 100644 index 00000000..0a53a0fe --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/STRUCTURE @@ -0,0 +1,25 @@ + + +key = original text + +data + 0..3 num of translation (n) + + translations + 0..3 num of file references (m) + 4..(4+m*4) file references + 4+m*4..first\0 translation + + +---------------------------------------------------- +Files structure. + + +"it" is the language + +catalogsinfo.it.db //Info about the catalog +keysindex.it.db //index of numofkeys->key +translations.it.db //guess.. (key->translation) +wordsindex.it.db //index of word->numofkey + + diff --git a/kbabel/kbabeldict/modules/dbsearchengine/TODO b/kbabel/kbabeldict/modules/dbsearchengine/TODO new file mode 100644 index 00000000..0e7fbd3a --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/TODO @@ -0,0 +1,27 @@ +TODO List fore KDE Database Translation Search Engine + +-Finish the port (use settings) done +-Improve settings (filter output) +-Database handling (and creation...) Done +-output filters +-fix fix fix... + +-delete item if putItem(nullitem) +-use matthias mask function for regexp +-remove bug of word split F&ormat is F ormat <- add remchar to charregexp + + +Major rewrite: +-New settings interface. +-addFunction from dbscan to database Done ? +-Edit function +-database manager +-download compendium from the internet + +Matthias ask: +- searching in translations, equivalent to searching in msgids (SearchEngine::startTranslationSearch(QString s) ) +- the absolute path of the files and maybe the number of the entry in the file ( maybe I will add this information to the TranslationInfo ) +- the possibility to get a list of all entries in a po file from the database. For this, I think, I will create a interface and send it to you. Maybe something like this: class TranslationDatabase { +public: // name is something like "kbabel.po" QList<CatalogItem> entriesOfFile(QString name); } + + diff --git a/kbabel/kbabeldict/modules/dbsearchengine/configure.in.bot b/kbabel/kbabeldict/modules/dbsearchengine/configure.in.bot new file mode 100644 index 00000000..58d6118f --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/configure.in.bot @@ -0,0 +1,5 @@ +if test "x$with_berkeley_db" = xcheck && test -z "$LIB_DBIV"; then + echo "" + echo "Dictionary plugin \"Translation Database\" for KBabel will not be built! Please install Berkeley Database IV. See http://www.sleepycat.com for more information.)" + echo "" +fi diff --git a/kbabel/kbabeldict/modules/dbsearchengine/configure.in.in b/kbabel/kbabeldict/modules/dbsearchengine/configure.in.in new file mode 100644 index 00000000..fbc3f1a8 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/configure.in.in @@ -0,0 +1,143 @@ +AC_ARG_WITH(berkeley-db, + [AC_HELP_STRING(--with-berkeley-db, + [enable the dictionary plugin based on Berkeley DB IV @<:@default=check@:>@])], + [], with_berkeley_db=check) + +LIB_DBIV="" +if test "x$with_berkeley_db" != xno; then + +AC_MSG_CHECKING([for Berkeley Database IV]) + +db_libraries="" +db_includes="" +db_name="" +ac_db_name="db" +ac_db_includes="" +ac_db_libraries="" +ac_db_include_file="db.h" + +AC_ARG_WITH(db-dir, + AC_HELP_STRING([--with-db-dir=DIR],[where the root of Berkeley DB IV is installed]), + [ ac_db_includes=-I"$withval"/include + ac_db_libraries=-L"$withval"/lib + ]) +AC_ARG_WITH(db-include-dir, + AC_HELP_STRING([--with-db-include-dir=DIR],[where the includes of Berkeley DB IV are installed]), + [ ac_db_includes=-I"$withval" + ]) +AC_ARG_WITH(db-include, + AC_HELP_STRING([--with-db-include=FILE],[path to the Berkeley DB IV header file]), + [ ac_db_include_file=-I"$withval" + ]) +AC_ARG_WITH(db-lib-dir, + AC_HELP_STRING([--with-db-lib-dir=DIR],[where the libs of Berkeley DB IV are installed]), + [ ac_db_libraries=-L"$withval" + ]) +AC_ARG_WITH(db-name, + AC_HELP_STRING([--with-db-name=NAME],[name of the Berkeley DB IV library (default db)]), + [ ac_db_name="$withval" + ]) + +AC_DEFUN([KDE_CHECK_DB_VERSION], +[ +ifelse($3,,,[LIBS="$kde_db_safe -l$3"]) +AC_TRY_LINK([ +#include <$2> +], +[ +#if DB_VERSION_MAJOR == 4 +DB *db; +#if DB_VERSION_MINOR > 0 +db->open( db, NULL, "test.db", NULL, DB_BTREE, DB_CREATE, 0644 ); +#else +db->open( db, "test.db", NULL, DB_BTREE, DB_CREATE, 0644 ); +#endif +#else +error +#endif +], +kde_cv_berk_database=$1 +) +]) + +AC_CACHE_VAL(kde_cv_berk_database, +[ +kde_safe_LDFLAGS=$LDFLAGS +kde_db_safe_LIBS=$LIBS +LDFLAGS="$LDFLAGS $X_LDFLAGS $USER_LDFLAGS $all_libraries $ac_db_libraries $all_includes $ac_db_includes" + +kde_cv_berk_database=NO +if test "xNO" = "x$kde_cv_berk_database" ; then + KDE_CHECK_DB_VERSION($ac_db_name, $ac_db_include_file, $ac_db_name) +fi +if test "xNO" = "x$kde_cv_berk_database" ; then + KDE_CHECK_DB_VERSION(db4, db4/db.h, db4) +fi +if test "xNO" = "x$kde_cv_berk_database" ; then + KDE_CHECK_DB_VERSION(db4-db, db.h, db4) +fi +if test "xNO" = "x$kde_cv_berk_database" ; then + KDE_CHECK_DB_VERSION(db, db.h, db) +fi + +LIBS=$kde_db_safe_LIBS +LDFLAGS=$kde_safe_LDFLAGS + +]) + +kde_db_header="" +DBSEARCHENGINE=dbsearchengine + +case "$kde_cv_berk_database" in + NO) + AC_MSG_RESULT(no) + LIB_DBIV="" + DBSEARCHENGINE="" + ;; + db) + LIB_DBIV="-l$ac_db_name" + kde_db_header=db.h + AC_MSG_RESULT(-l$ac_db_name) + AC_DEFINE_UNQUOTED(HAVE_DB_DB_H, 1, [DB 4 header location] ) + ;; + db4-db) + LIB_DBIV='-ldb4' + kde_db_header=db.h + AC_MSG_RESULT("flag is -ldb4 and header is db.h") + AC_DEFINE_UNQUOTED(HAVE_DB_DB_H, 1, [DB 4 header location] ) + ;; + db4) + LIB_DBIV='-ldb4' + kde_db_header=db4/db.h + AC_MSG_RESULT(-ldb4) + AC_DEFINE_UNQUOTED(HAVE_DB4_DB_H, 1, [DB 4 header location] ) + ;; + $ac_db_name) + LIB_DBIV="-l$ac_db_name" + kde_db_header="$ac_db_include_file" + AC_MSG_RESULT(user specified $ac_db_name) + if test "x$ac_db_include_file" = "xdb.h" ; then + AC_DEFINE_UNQUOTED(HAVE_DB_DB_H, 1, [DB 4 header location] ) + else + AC_DEFINE_UNQUOTED(USE_DB_H_PATH, <$ac_db_include_file>, [DB 4 header path]) + fi + + ;; +esac + +DBIV_LDFLAGS="$ac_db_libraries" +DBIV_INCLUDES="$ac_db_includes" +DBIV_NAME="$ac_db_name" + +if test "x$with_berkeley_db" != xcheck && test -z "$LIB_DBIV"; then + AC_MSG_ERROR([--with-berkeley-db was given, but test for Berkeley DB IV failed]) +fi +fi + +AC_SUBST(DBIV_INCLUDES) +AC_SUBST(DBIV_LDFLAGS) +AC_SUBST(DBIV_NAME) + +AC_SUBST(LIB_DBIV) + +AM_CONDITIONAL(include_DBSEARCHENGINE, test -n "$DBSEARCHENGINE") diff --git a/kbabel/kbabeldict/modules/dbsearchengine/database.cpp b/kbabel/kbabeldict/modules/dbsearchengine/database.cpp new file mode 100644 index 00000000..003a3a95 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/database.cpp @@ -0,0 +1,1533 @@ +/*************************************************************************** + database.cpp - + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + email : rizzi@kde.org + ***************************************************************************/ + +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + Copyright 2003 Stanislav Visnovsky visnovsky@kde.org + + License GPL v 2.0 + + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * + +*/ +#include <stdlib.h> +#include <kdebug.h> +#include <string.h> +#include <resources.h> +#include "database.h" +#include <unistd.h> + +#include <qfile.h> + +#include <ktempfile.h> +#include <kio/netaccess.h> + +WordItem::WordItem (char *data, QString w) +{ + word = w; + + count = *(uint32 *) data; + + data += 4; +//score=*(int *)data; + data += 4; + + + locations = (uint32 *) malloc (4 * count); + memcpy (locations, data, 4 * count); + + +} + +WordItem::WordItem (QString w) +{ + locations = NULL; + count = 0; + word = w; + score = -1; // it means no references found. +} + +/* + +WordItem::WordItem(const WordItem &wi) +{ + count=wi.count; + score=wi.score; + word=wi.word; + locations.duplicate(wi.locations); + locations.detach(); +} + +WordItem& WordItem::operator=(const WordItem & wi ) +{ +WordItem *i=new WordItem(wi); +i->locations.detach(); +return *i; +} +*/ + +bool +WordItem::notFound () +{ + return (score == -1); +} + +InfoItem::InfoItem () +{ + catalogName = "No catalog"; + lastTranslator = "No translator"; + lastFullPath = ""; + charset = "No charset"; + language = "No language"; +} + +InfoItem::InfoItem (const char *rawData, QString lang) +{ + const char *rd; + rd = rawData; + int len; + unsigned int secs; + // I'll change the charset handling if needed + + charset = "Utf8"; + + catalogName = QString::fromUtf8 (rd); + len = strlen (rd) + 1; + rd += len; + lastTranslator = QString::fromUtf8 (rd); + len = strlen (rd) + 1; + rd += len; + secs = *(unsigned int *) rd; + revisionDate.setTime_t (secs); + rd += 4; + lastFullPath = QString::fromUtf8 (rd); + len = strlen (rd) + 1; + rd += len; + + + language = lang; +} + +void +InfoItem::rawData (char *rawData) +{ + char *rd; + rd = rawData; + + strcpy (rd, catalogName.utf8 ()); + rd += strlen (rd) + 1; + strcpy (rd, lastTranslator.utf8 ()); + rd += strlen (rd) + 1; + + + //QDate Time problem!!!!!!!!!!! + QDateTime zeroDate; + zeroDate.setTime_t (0); + *(unsigned int *) rd = -revisionDate.secsTo (zeroDate); + rd += 4; + strcpy (rd, lastFullPath.utf8 ()); + rd += strlen (rd) + 1; + + *rd = 0; //Empty for further info + +} + +int +InfoItem::size () +{ + int _size; + _size = 0; + _size += 1; // 1 Empty field; + _size += 3; // Terminating \0 of next 3 strings + _size += 4; // Int size (date) + _size += strlen (catalogName.utf8 ()); + _size += strlen (lastTranslator.utf8 ()); + _size += strlen (lastFullPath.utf8 ()); + + return _size; +} + +// this is a quick hack to copy a local file +int +copy_hack (QFile & input, QFile & output) +{ + if (!input.isOpen ()) + { + if (!input.open (IO_ReadOnly)) + return -1; + } + + if (!output.isOpen ()) + { + if (!output.open (IO_WriteOnly)) + return -1; + } + + char buffer[10240]; + int s = 0; + while (!input.atEnd ()) + { + s = input.readBlock (buffer, 10240); + output.writeBlock (buffer, s); + } + output.close (); + input.close (); + return 0; +} + +DataBaseItem::DataBaseItem () +{ + numTra = 0; + location = 0; +} + +DataBaseItem::DataBaseItem (char *_key, char *_data) +{ + + char *data = _data; + key = QString::fromUtf8 (_key); + + unsigned int i, r; + numTra = *(uint32 *) data; + data += 4; + location = *(uint32 *) data; + data += 4; + + for (i = 0; i < numTra; i++) + { + + TranslationItem tr; + tr.numRef = *(uint32 *) data; + + data += 4; + for (r = 0; r < tr.numRef; r++) + { + + int ref; + ref = *(uint32 *) data; + data += 4; + + tr.infoRef.append (ref); + } + tr.translation = QString::fromUtf8 ((const char *) data); + translations.append (tr); + data += strlen (data) + 1; + + } + +} + +uint32 +DataBaseItem::sizeKey () +{ + return strlen (key.utf8 ()) + 1; +} + +uint32 +DataBaseItem::sizeData () +{ + unsigned int i, _size = 4; + _size += numTra * 4; + _size += 4; // location + for (i = 0; i < numTra; i++) + { + _size += strlen (translations[i].translation.utf8 ()) + 1; // +1 is for \0 + _size += translations[i].numRef * 4; + } + return _size; +} + +void +DataBaseItem::toRawKey (char *_key) +{ + strcpy (_key, key.utf8 ()); +} + +void +DataBaseItem::toRawData (char *_data) +{ + char *data = _data; + unsigned int i, r; + + *(uint32 *) data = numTra; + + data += 4; + + *(uint32 *) data = location; + data += 4; + + for (i = 0; i < numTra; i++) + { + TranslationItem tr (translations[i]); + *(uint32 *) data = tr.numRef; + data += 4; + for (r = 0; r < tr.numRef; r++) + { + *(uint32 *) data = tr.infoRef[r]; //sub i with r + + data += 4; + } + strcpy ((char *) data, tr.translation.utf8 ()); + data += strlen (tr.translation.utf8 ()) + 1; + } + +} + + +DataBaseManager::DataBaseManager (QString directory, QString lang, + QObject * parent, const char *name): +QObject (parent, name) +{ + QString filename; + + language = lang; + iAmOk = true; + basedir = directory; + indexDb = wordDb = infoDb = db = 0; + openDataBase (); + + +} + +void +DataBaseManager::openDataBase () +{ + kdDebug () << "Opendatabase" << endl; + QString directory; + directory = basedir; + QString ll = "." + language; + if (ll == ".") + ll = ".NOLANG"; + + QString transfilename = "%1/translations%2.db"; + transfilename = transfilename.arg (directory).arg (ll); + + QString infofilename = "%1/catalogsinfo%2.db"; + infofilename = infofilename.arg (directory).arg (ll); + + QString wordsfilename = "%1/wordsindex%2.db"; + wordsfilename = wordsfilename.arg (directory).arg (ll); + + QString keysfilename = "%1/keysindex%2.db"; + keysfilename = keysfilename.arg (directory).arg (ll); + + cursor = 0; + int ret; + + if (!db) + db_create (&db, 0, 0); + + db_create (&infoDb, 0, 0); + db_create (&wordDb, 0, 0); + db_create (&indexDb, 0, 0); + + ret = db->open (db, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + transfilename.local8Bit (), 0, DB_BTREE, 0, 0644); + + if (ret == DB_OLD_VERSION) + { + kdDebug (KBABEL_SEARCH) << "Trying upgrade" << endl; + // try upgrade + + KTempFile transFile, infoFile, keysFile, wordsFile; + + // do the upgrade on the translation file + QFile transfilenameFile (transfilename); + + if ((ret = copy_hack (transfilenameFile, *transFile.file ())) == 0) + { + ret = db->upgrade (db, transFile.name ().local8Bit (), 0); + } + + if (ret != 0) + { + kdDebug (KBABEL_SEARCH) << "Cannot upgrade translations, " << + ret << endl; + // cleanup + transFile.unlink (); + iAmOk = false; + emit cannotOpenDB (ret); + return; + } + + // do the upgrade on the info file + QFile infofilenameFile (infofilename); + if ((ret = copy_hack (infofilenameFile, *infoFile.file ())) == 0) + { + ret = + infoDb->upgrade (infoDb, infoFile.name ().local8Bit (), + 0); + } + + if (ret != 0) + { + kdDebug (KBABEL_SEARCH) << "Cannot upgrade catalogsinfo" << + endl; + // cleanup + transFile.unlink (); + infoFile.unlink (); + iAmOk = false; + emit cannotOpenDB (ret); + return; + } + + // do the upgrade on the words file + QFile wordfilenameFile (wordsfilename); + if ((ret = copy_hack (wordfilenameFile, *wordsFile.file ())) == 0) + { + ret = + wordDb->upgrade (wordDb, wordsFile.name ().local8Bit (), + 0); + } + + if (ret != 0) + { + kdDebug (KBABEL_SEARCH) << "Cannot upgrade words" << endl; + // cleanup + transFile.unlink (); + infoFile.unlink (); + wordsFile.unlink (); + iAmOk = false; + emit cannotOpenDB (ret); + return; + } + + // do the upgrade on the keys file + QFile keysfilenameFile (keysfilename); + if ((ret = copy_hack (keysfilenameFile, *keysFile.file ())) == 0) + { + ret = + indexDb->upgrade (indexDb, keysFile.name ().local8Bit (), + 0); + } + + if (ret != 0) + { + kdDebug (KBABEL_SEARCH) << "Cannot upgrade keys" << endl; + // cleanup + transFile.unlink (); + infoFile.unlink (); + wordsFile.unlink (); + keysFile.unlink (); + iAmOk = false; + emit cannotOpenDB (ret); + return; + } + + kdDebug (KBABEL_SEARCH) << "Files upgraded, copying" << endl; + // use temporary file instead + if (ret == 0) + { + KIO::NetAccess::del (KURL::fromPathOrURL (transfilename)); + copy_hack (*transFile.file (), transfilenameFile); + transFile.unlink (); + + KIO::NetAccess::del (KURL::fromPathOrURL (infofilename)); + copy_hack (*infoFile.file (), infofilenameFile); + infoFile.unlink (); + + KIO::NetAccess::del (KURL::fromPathOrURL (wordsfilename)); + copy_hack (*wordsFile.file (), wordfilenameFile); + wordsFile.unlink (); + + KIO::NetAccess::del (KURL::fromPathOrURL (keysfilename)); + copy_hack (*keysFile.file (), keysfilenameFile); + keysFile.unlink (); + + ret = db->open (db, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + transfilename.local8Bit (), 0, DB_BTREE, 0, + 0644); + if (ret != 0) + { + kdWarning (KBABEL_SEARCH) << + "transFilename database can't be opened." << endl; + kdWarning (KBABEL_SEARCH) << + "Please, report this incident and how to reproduce it to kbabel@kde.org." + << endl; + iAmOk = false; + emit cannotOpenDB (ret); + return; + } + + } + kdDebug (KBABEL_SEARCH) << "Upgrade done OK" << endl; + } + +// Open catalogs information database + + + + ret = infoDb->open (infoDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + infofilename.local8Bit (), 0, DB_RECNO, 0, 0644); + if (ret != 0) + { + iAmOk = false; + emit cannotOpenDB (ret); + //Process error here. + } + else + loadInfo (); + + + +// Words index database + + ret = wordDb->open (wordDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + wordsfilename.local8Bit (), 0, DB_BTREE, 0, 0644); + if (ret != 0) + { + iAmOk = false; + emit cannotOpenDB (ret); + //Process error here. + } + +//Index of keys. + + ret = indexDb->open (indexDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + keysfilename.local8Bit (), 0, DB_RECNO, 0, 0644); + if (ret != 0) + { + iAmOk = false; + emit cannotOpenDB (ret); + //Process error here. + } + + +} + +void +DataBaseManager::closeDataBase () +{ + if (iAmOk) + { + db->sync (db, 0); + db->close (db, 0); + + infoDb->sync (infoDb, 0); + infoDb->close (infoDb, 0); + + wordDb->sync (wordDb, 0); + wordDb->close (wordDb, 0); + + indexDb->sync (indexDb, 0); + indexDb->close (indexDb, 0); + + // can not be opened again + indexDb = wordDb = infoDb = db = 0; + + } + +} + + +// I'm not sure this is a good function !!! + +void +DataBaseManager::sync () +{ +// if(iAmOk) +// { +// db->sync(db,0); +// infoDb->sync(infoDb,0); +// cursor=0; +// } + + // closeDataBase(); +// openDataBase(); + + db->sync (db, 0); + infoDb->sync (infoDb, 0); + wordDb->sync (wordDb, 0); + indexDb->sync (indexDb, 0); + loadInfo (); +} + + +DataBaseManager::~DataBaseManager () +{ + closeDataBase (); +} + +int +DataBaseManager::putItem (DataBaseItem * item, bool ow) +{ + DBT key, data; + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + bool ret; + + uint32 loc = 0; + if (item->location == 0) + { + loc = item->location = appendKey (item->key); +// kdDebug(0) << "New key " << item->location << endl; + } + key.size = item->sizeKey (); + data.size = item->sizeData (); + + key.data = malloc (key.size); + data.data = malloc (data.size); + + + item->toRawKey ((char *) key.data); + item->toRawData ((char *) data.data); + + + if (ow) + ret = db->put (db, 0, &key, &data, 0); + else + ret = db->put (db, 0, &key, &data, DB_NOOVERWRITE); + +//check ret + + if (loc != 0) //I'm new! + { + uint32 location = loc; + + QValueList < QString > wlist; + + wlist = wordsIn (item->key); + + QValueList < QString >::Iterator wlistit; + + for (wlistit = wlist.begin (); wlistit != wlist.end (); ++wlistit) + { + addLocation (*wlistit, location); + } + + } + + free (key.data); + free (data.data); //READ DOCU !!!! + + return ret; + +} + +DataBaseItem +DataBaseManager::getItem (QString key) +{ + if (!iAmOk) + return DataBaseItem (); + + + DBT _key, data; + + memset (&_key, 0, sizeof (DBT)); + + memset (&data, 0, sizeof (DBT)); + + int len = strlen (key.utf8 ()); + _key.data = malloc (len + 1); + _key.size = len + 1; + strcpy ((char *) _key.data, key.utf8 ()); + + + int ret; + ret = db->get (db, 0, &_key, &data, 0); + + if (ret != 0) + { + free (_key.data); + return DataBaseItem (); //return an empty database item + } + + DataBaseItem returnItem = + DataBaseItem ((char *) _key.data, (char *) data.data); + + free (_key.data); + return returnItem; + +} + + + + +DataBaseItem +DataBaseManager::cursorGet (uint32 flags) +{ + + if (!iAmOk) + return DataBaseItem (); + int re; + DBT key, data; + + memset (&key, 0, sizeof (DBT)); + + memset (&data, 0, sizeof (DBT)); + + if (cursor == 0) + re = db->cursor (db, 0, &cursor, 0); + + int ret; + if ((ret = cursor->c_get (cursor, &key, &data, flags)) == 0) + { + return DataBaseItem ((char *) key.data, (char *) data.data); + } + else + { + kdDebug (KBABEL_SEARCH) << QString ("...cursor getting...%1"). + arg (ret) << endl; + + return DataBaseItem (); + } +} + + +DataBaseItem +DataBaseManager::firstItem () +{ + return cursorGet (DB_FIRST); +} + +DataBaseItem +DataBaseManager::currentItem () +{ + return cursorGet (DB_CURRENT); +} + + +DataBaseItem +DataBaseManager::nextItem () +{ + return cursorGet (DB_NEXT); +} + + +bool +DataBaseManager::isOk () +{ + return iAmOk; +} + +int +DataBaseManager::count () +{ + DB_BTREE_STAT *dstat = 0; +#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3 + db->stat (db, NULL, &dstat, DB_FAST_STAT); +#else + db->stat (db, &dstat, DB_FAST_STAT); +#endif + int ret = dstat->bt_nkeys; + free (dstat); + + return ret; +} + +int +DataBaseManager::current () +{ +// THIS FUNCTION SEEM TO NOT WORK (not used) + if (!iAmOk) + return 0; + + DBT key, data; + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + if (cursor != 0) + db->cursor (db, 0, &cursor, 0); + cursor->c_get (cursor, &key, &data, DB_GET_RECNO); + return *(uint32 *) (data.data); + +} + +int +DataBaseManager::createDataBase (QString directory, + QString language, int mode) +{ + QString filename; + QString ll = "." + language; + if (ll == ".") + ll = ".NOLANG"; + filename = "%1/translations%2.db"; + filename = filename.arg (directory).arg (ll); + + rename (filename.local8Bit (), filename.local8Bit () + ",old"); + +//kdDebug(0) << QString("Creating %1").arg(filename) << endl; + + iAmOk = true; + + int ret; + + if (!db) + { + if (db_create (&db, 0, 0) != 0) + { + kdDebug() << "db_create db failed" << endl; + iAmOk = false; + return false; + } + } + + db->set_flags (db, DB_RECNUM); + ret = db->open (db, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + filename.local8Bit (), 0, DB_BTREE, DB_CREATE | DB_EXCL, + mode); + if (ret != 0) + { + kdDebug() << "db->open " << filename << " " << mode << " failed" << endl; + iAmOk = false; + } + + + filename = "%1/catalogsinfo%2.db"; + filename = filename.arg (directory).arg (ll); + rename (filename.local8Bit (), filename.local8Bit () + ",old"); + + db_create (&infoDb, 0, 0); + ret = infoDb->open (infoDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + filename.local8Bit (), 0, DB_RECNO, DB_CREATE, mode); + if (ret != 0) + iAmOk = false; + + + + filename = "%1/wordsindex%2.db"; + filename = filename.arg (directory).arg (ll); + rename (filename.local8Bit (), filename.local8Bit () + ",old"); + + db_create (&wordDb, 0, 0); + ret = wordDb->open (wordDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + filename.local8Bit (), 0, DB_BTREE, DB_CREATE, mode); + if (ret != 0) + iAmOk = false; + + + + filename = "%1/keysindex%2.db"; + filename = filename.arg (directory).arg (ll); + rename (filename.local8Bit (), filename.local8Bit () + ",old"); + + db_create (&indexDb, 0, 0); + ret = indexDb->open (indexDb, +#if DB_VERSION_MINOR > 0 + NULL, +#endif + filename.local8Bit (), 0, DB_RECNO, DB_CREATE, mode); + if (ret != 0) + iAmOk = false; + + + + if (iAmOk) + loadInfo (); + else + kdDebug (KBABEL_SEARCH) << QString ("I am NOT ok : %1"). + arg (ret) << endl; + +//THIS IS WRONG, rewrite the error handling. + return iAmOk; + +} + +InfoItem +DataBaseManager::getCatalogInfo (int n) +{ + + DBT key; + DBT data; + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + key.data = &n; + key.size = 4; + +//Check for errors + int ret = infoDb->get (infoDb, 0, &key, &data, 0); //DB_SET_RECNO); + + if (ret) + { + return InfoItem (); + } + +// kdDebug(0) << QString("Trad %1").arg(ret) << endl; + + InfoItem it ((char *) data.data, language); +//free(data.data); // Read docu for this!!!! + + return it; + +} + +int +DataBaseManager::addCatalogInfo (InfoItem * catInfo, int cat = -1) +{ + DBT data; + DBT key; + + // clean up data + memset (&data, 0, sizeof (DBT)); + memset (&key, 0, sizeof (DBT)); + + int ret = 0, err; + if (cat >= 0) + ret = cat; + key.size = 4; + key.data = &ret; + data.size = catInfo->size (); + data.data = malloc (data.size); + + catInfo->rawData ((char *) data.data); + + // store the catalog data into database + if (cat >= 0) + err = infoDb->put (infoDb, 0, &key, &data, 0); + else + err = infoDb->put (infoDb, 0, &key, &data, DB_APPEND); + + + ret = *(int *) key.data; + + // Append to the list of catalogInfo + info.append (*catInfo); + + // cleanup unneeded data memory + free (data.data); + return ret; +} + +int +DataBaseManager::searchCatalogInfo (QString location) +{ + int n = 0; + QValueList < InfoItem >::Iterator it; + for (it = info.begin (); it != info.end (); ++it) + { + n++; + if ((*it).catalogName == location) + return n; + } + return -1; +} + +bool +DataBaseManager::putCatalogInfo (int refnum, InfoItem * catInfo) +{ + DBT data; + DBT key; + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + int ret; + key.size = 4; + key.data = &refnum; + + data.size = catInfo->size (); + data.data = malloc (data.size); + catInfo->rawData ((char *) data.data); + + ret = infoDb->put (infoDb, 0, &key, &data, 0); + + free (data.data); + + return (ret == 0); +} + +void +DataBaseManager::loadInfo () +{ + int nrec; + DB_BTREE_STAT *stat; +// memset(&stat,0,sizeof(DB_BTREE_STAT)); +#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR >= 3 + if (infoDb->stat (infoDb, NULL, &stat, DB_FAST_STAT)) + fprintf (stderr, "Cannot stat\n"); +#else + if (infoDb->stat (infoDb, &stat, DB_FAST_STAT)) + fprintf (stderr, "Cannot stat\n"); +#endif + nrec = stat->bt_nkeys; + free (stat); + + info.clear (); + for (int i = 1; i <= nrec; i++) // I think DB2 Recno are 1 based. + { + info.append (getCatalogInfo (i)); + } + +} + + +QValueList < QString > DataBaseManager::wordsIn (QString string) +{ + QString + a; + QValueList < QString > words; + int + i, + l; + + a = string.simplifyWhiteSpace (); + a = a.stripWhiteSpace (); + a = a.lower (); + l = a.length (); + + int + c = 0; + //words.setAutoDelete(true); //Not sure... check if it is en. + QString + m; + for (i = 0; i < l; i++) + if (a[i].isLetterOrNumber ()) + { + m += a[i]; + } + else if (a[i].isSpace ()) + { + words.append (m); + c++; // C++ ;-) + m = ""; + } + words.append (m); + + return words; +} + + + +WordItem +DataBaseManager::getWordLocations (QString word) +{ + + QString keystring = word.lower (); + + DBT key; + DBT data; + + char *keydata; + + int intlen = strlen (keystring.utf8 ()); + + keydata = (char *) malloc (intlen + 1); + strcpy (keydata, keystring.utf8 ()); + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + key.data = keydata; + key.size = intlen + 1; + +//Check for errors + int ret = wordDb->get (wordDb, 0, &key, &data, 0); //DB_SET_RECNO); +//MAYBE THIS WORD IS NO WHERE!! + if (ret) + { + free (keydata); + return WordItem (keystring); + } + + + WordItem it ((char *) data.data, keystring); + + free (keydata); + + +// kdDebug(0) << ((uint32 *)it.locations.data())[0] << endl; + + return it; + +} + + +bool +DataBaseManager::addLocation (QString word, unsigned int location) +{ + + QString keystring = word.lower (); + + + DBT key; + DBT data; + + char *keydata; + char *newdata; + + int intlen = strlen (keystring.utf8 ()); + + keydata = (char *) malloc (intlen + 1); + strcpy (keydata, keystring.utf8 ()); + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + key.data = keydata; + key.size = intlen + 1; + + strcpy ((char *) key.data, keystring.utf8 ()); + +//Check for errors + int ret = wordDb->get (wordDb, 0, &key, &data, 0); //DB_SET_RECNO); + +//Check if you get something or not + if (ret == 0) // CHANGE IT!!!!! if found something + { + uint32 *d; + d = (uint32 *) data.data; + uint32 num = d[0]; + uint32 loc = 0; //Position from 0 to num-1 + int totalrecord = count (); + uint32 step = 1; + + int antibounce = 0; +//d+=4; +//int score=d[1]; +//d+=4; + bool forward, end = false; + + d[1] = 0; //Before the begin... + + d += 2; //1 uint32! + + +//Try to guess... + loc = location * num / totalrecord + 1; + if (loc >= num) + loc = num - 1; + + + + +//Continue if next is smaller or if previous is greater +//before the while check if it is ok + if (loc == 0) + { + if (d[loc] > location) + end = true; + else + loc = 1; + } + + if ((loc == num) && !end) + { + if (d[loc - 1] < location) + end = true; + else + loc = num - 1; + } + + + + while ((!end) && ((forward = (d[loc]) < location) + || ((loc > 0) && (d[loc - 1] > location)))) + { + + antibounce++; + //calculate step or use antibounce + if (abs ((int) d[loc] - (int) location) < 50 + || antibounce > 100) + step = 1; //Go linear... + else + { + step = + (abs (d[loc] - location) * num) / totalrecord + 1; + + } + + kdDebug (KBABEL_SEARCH) << "Insert:" << location << + " We are at: " << loc << " i.e. " << d[loc] << " Step:" + << step << endl; + if (loc > 0) + { + if (forward) + kdDebug (KBABEL_SEARCH) << "Go " << "forward" << + " prev is " << d[loc - 1] << endl; + else + kdDebug (KBABEL_SEARCH) << "Go " << "backward" << + " prev is " << d[loc - 1] << endl; + } + + if (forward) + { + if (loc + step < num) + loc += step; // Go forward + else + loc = num; // Go to num + } + else + { + if (loc > step) + loc -= step; //Go backward + else + loc = 0; // Go to 0 + } + + //Now check if I am in the right place [THIS IS NOT NECESSARY] + + //check if loc and loc -1 are well defined 1<loc<num-1 + if (loc > num) + loc = num; //Must not happen, idem + + if (loc == 0) + { + if (d[loc] > location) + end = true; + else + loc = 1; + } + + if ((loc == num) && !end) + { + if (d[loc - 1] < location) + end = true; + else + loc = num - 1; + } + + } + + if (loc == num) + kdDebug (KBABEL_SEARCH) << "END" << endl; + if (loc == 0) + kdDebug (KBABEL_SEARCH) << "BEGIN" << endl; + + if (loc > 0) + kdDebug (KBABEL_SEARCH) << location << " inserted between " << + d[loc - 1] << " and " << d[loc] << endl; + + if ((loc < num && location == d[loc]) || (loc > 0 && location == d[loc - 1])) //What about word repetition + { + free (keydata); + return true; //Why true ?? + } + + + +//Now insert and put back in the database! + newdata = (char *) malloc (4 * (num + 3)); //uint32*(num+score+1..NUM+new) + memcpy (newdata, data.data, 4 + 4 + 4 * loc); + char *secondpart = (char *) data.data; + secondpart += 4 * (loc + 2); + memcpy ((newdata + 4 * (loc + 3)), secondpart, (num - loc) * 4); + uint32 *intdata = (uint32 *) newdata; + intdata[0] = num + 1; + //ADD HERE code to recalc score + intdata[loc + 2] = location; + +//ok send it to database! + memset (&data, 0, sizeof (DBT)); + + data.data = newdata; + data.size = 4 * (3 + num); + } //found sounthing + else + { //found nothing + newdata = (char *) malloc (4 * 3); + uint32 *intdata = (uint32 *) newdata; + intdata[0] = 1; + intdata[1] = 1; + intdata[2] = location; + memset (&data, 0, sizeof (DBT)); + data.data = newdata; + data.size = 4 * 3; + } + + memset (&key, 0, sizeof (DBT)); +//memset(&data,0,sizeof(DBT)); + + key.data = keydata; + key.size = intlen + 1; + + ret = wordDb->put (wordDb, 0, &key, &data, 0); //DB_SET_RECNO); + + + free (newdata); + free (keydata); +//return it; + + return true; +} + + +bool +DataBaseManager::removeLocation (QString /*word */ , int /*location */ ) +{ + +//#warning TODO: REMOVE LOCATION + return true; + +} + +uint32 +DataBaseManager::appendKey (QString _key) +{ + + DBT key; + DBT data; + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + + uint32 ret = 0, err; + key.size = 4; + key.data = &ret; + + data.size = strlen (_key.utf8 ()) + 1; + data.data = malloc (data.size); + + strcpy ((char *) data.data, _key.utf8 ()); + + err = indexDb->put (indexDb, 0, &key, &data, DB_APPEND); + + if (err) + ret = 0; + else + ret = *(uint32 *) key.data; + +//kdDebug(0) << QString("Append result %1,err = %1").arg(ret).arg(err) << endl; + + + free (data.data); + + return ret; + +} + +QString +DataBaseManager::getKey (uint32 n) +{ + + DBT key; + DBT data; + + memset (&key, 0, sizeof (DBT)); + memset (&data, 0, sizeof (DBT)); + + key.data = &n; + key.size = 4; + +//Check for errors + int ret = indexDb->get (indexDb, 0, &key, &data, 0); //DB_SET_RECNO); + if (ret) + return QString::null; + + return QString::fromUtf8 ((char *) data.data); + +// kdDebug(0) << QString("Trad %1").arg(ret) << endl; + +} + +int +DataBaseManager::catalogRef (QString location, QString author, QString path) +{ + InfoItem cinfo; + int cat, catnum; + cat = searchCatalogInfo (location); + + if (cat == -1) //Not exist + { + cinfo.catalogName = location; + cinfo.lastTranslator = author; + cinfo.lastFullPath = path; + //TO DO: + // //Add date info + + kdDebug (0) << "New catalog " << endl; + catnum = addCatalogInfo (&cinfo); + //sync(); + kdDebug (0) << "Ref " << catnum << endl; + } + else + { + cinfo = getCatalogInfo (cat); + //Update date. + + //last path must be updated + cinfo.lastFullPath = path; + kdDebug (0) << "New full path " << path << endl; + kdDebug (0) << "Ref " << cat << endl; + + catnum = addCatalogInfo (&cinfo, cat); + kdDebug (0) << " must be equal to " << catnum << endl; + + catnum = cat; + } + + return catnum; +} + +int +DataBaseManager::putNewTranslation (QString key, QString tran, int catalog, + bool ow) +{ + int catnum = catalog; + int count = 0; + QString msgid = key; + DataBaseItem dbit = getItem (msgid); + + if (dbit.numTra == 0) //Not found + { + dbit.numTra += 1; + + // use local variable, dbit.translations is QValueList and + // will create own copy + TranslationItem tra; + tra.numRef = 1; + tra.translation = tran; + tra.infoRef.append (catnum); + dbit.translations.append (tra); + dbit.key = key; + + //Check ret value + count++; + + int aa = putItem (&dbit); + if (aa) + kdDebug (0) << QString ("-----------put code ") << aa << endl; + } + else + { + // key exists + + QString msgstr = tran; + bool found_catalog_info = false, foundTr = false, isThisOne = false; + + QValueList < TranslationItem >::Iterator ittr; + bool rem = false; + + // check all translations in the list + for (ittr = dbit.translations.begin (); + ittr != dbit.translations.end (); rem ? ittr : ++ittr) + { + rem = false; + found_catalog_info = false; + + // is the translation one we should put there? + isThisOne = (*ittr).translation == msgstr; + + // is there the catnum we are looking for? + if ((*ittr).infoRef.find (catnum) != (*ittr).infoRef.end ()) + { + found_catalog_info = true; + if (ow && !isThisOne) + { + // I'll look for my catalog reference to del old + kdDebug (0) << "Removing the old translation " << endl; + (*ittr).numRef -= 1; + (*ittr).infoRef.remove (catnum); + if ((*ittr).numRef == 0) + { + dbit.numTra -= 1; + // point the iterator to the next valid item + ittr = dbit.translations.remove (ittr); + rem = true; + } + } + } + + if (isThisOne) + { + if (!found_catalog_info) + { + //There are no reference of this catalog for this translation => add it + (*ittr).infoRef.append (catnum); + (*ittr).numRef += 1; + } + foundTr = true; // Ok, we found this translation, no need to add it. + } + } + + if (!foundTr) //This translation is new => Add it ! + { + count++; + TranslationItem tra; + tra.numRef = 1; + tra.translation = msgstr; + tra.infoRef.append (catnum); + + dbit.translations.append (tra); + dbit.numTra += 1; + } + + //put the new item in database overwriting the old one. + //Check ret value + int aa = putItem (&dbit, true); + if (aa) + kdDebug (0) << QString ("-----------put code ") << aa << endl; + + } + return count; +} + + +void +DataBaseManager::rebuildIndexes () +{ +// uint32 loc; +#if 0 +//Reset the 2 databases here. + + while (0 /*browse keys here */ ) + { + loc = item->location = appendKey (item->key); + + uint32 location = loc; + + QValueList < QString > wlist; + + wlist = wordsIn (item->key); + + QValueList < QString >::Iterator wlistit; + + for (wlistit = wlist.begin (); wlistit != wlist.end (); ++wlistit) + { + addLocation (*wlistit, location); + } + + } +#endif +} + + + +#include "database.moc" diff --git a/kbabel/kbabeldict/modules/dbsearchengine/database.h b/kbabel/kbabeldict/modules/dbsearchengine/database.h new file mode 100644 index 00000000..bf4767df --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/database.h @@ -0,0 +1,329 @@ +/*************************************************************************** + database.h - + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + email : rizzi@kde.org + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * + ***************************************************************************/ +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + +*/ + + +#ifndef _DATABASE_H_ +#define _DATABASE_H_ + +#include <config.h> + +#ifdef USE_DB_H_PATH +#include USE_DB_H_PATH +#else +#ifdef HAVE_DB4_DB_H +#include <db4/db.h> +#else +#include <db.h> +#endif +#endif + +#define uint32 u_int32_t + +#include <qvaluelist.h> +#include <qptrlist.h> +#include <qstring.h> +#include <qobject.h> +#include <qdatetime.h> +#include <qcstring.h> //bytearray + + +class WordItem // Returned by getWordLocations +{ + public: + + WordItem(char *data,QString w); + WordItem(QString w); + //WordItem(const WordItem &wi); + //WordItem& operator=(const WordItem & wi ); + + //The word (key in database) + QString word; + //Sorted locations + //QByteArray locations; //too many problems with this.. + //NOTE: + //This is allocated only if you call WordItem(char *data,QString w); + //YOU MUST FREE IT when you do not need it anymore + //No destructor will do it !!! + + uint32 *locations; + + + uint32 count; + //Is this word common ? + int score; + + bool notFound(); +}; + + +class InfoItem +{ + public: + + //Create the NO INFO item + InfoItem(); + + // Create an info item from raw data + InfoItem(const char *rawData,QString lang); + + QString catalogName; + QString lastFullPath; + + QString lastTranslator; + QDateTime revisionDate; + QString charset; + QString language; + + + int size(); + void rawData(char *); + +}; + +class TranslationItem +{ + public: + QString translation; + QValueList<int> infoRef; + uint32 numRef; + +}; + +class DataBaseItem +{ + public: + /* + Create empty class; + */ + DataBaseItem(); + + /* + Create a DataBaseItem from raw data + */ + DataBaseItem(char *_key,char *_data); + + /* + return the size (in raw data) of this item. + */ + uint32 sizeData(); + uint32 sizeKey(); + + bool isNull() { return (numTra==0); } + + /* + You MUST allocate data of sizeData() byte. + */ + void toRawData(char *_data); + void toRawKey(char *_key); + + QString key; + QValueList<TranslationItem> translations; + uint32 numTra; + uint32 location; +}; + +class DataBaseManager : public QObject +{ + + Q_OBJECT + + public: + /* + Main constructor. + directory is the name of the directory where the database structre + is stored. + + DIRECTORY/translations.$LANG.db The trasnaltion archive + DIRECTORY/catalogsinfo.$LANG.db Info about catalogs + + //Not yet implemented + DIRECTORY/wordsindex.$LANG.db Index of words + + + */ + + DataBaseManager(QString directory,QString lang,QObject *parent=0,const char *name=0); + ~DataBaseManager(); + /* + Create a new databse structure. + + */ + int createDataBase(QString directory,QString language,int mode=0664); + + + + + /* + Put a DataBaseItem into the database. + + if ow==false enter the new DataBaseItem only + if the key of DataBaseItem does not previously exist + + It also update the wordIndex. + + */ + + int putItem(DataBaseItem *item,bool ow=false); + + DataBaseItem getItem(QString key); + + /* + @return the first entry in the database. + */ + + DataBaseItem firstItem(); + + /* + @return the current entry of the database. + */ + + DataBaseItem currentItem(); + + /* + @return the next entry in the database. + */ + + DataBaseItem nextItem(); + + + /* + * Add a new translation to the database + * catalog is a valid catalog refnum (use catalogRef to get one) + * if ow is true a translation of a key coming from catalog is + * overwritten if you provide a new translation + */ + + int putNewTranslation(QString key,QString tran,int catalog,bool ow=true); + + /* + @return info about catalog n + */ + + InfoItem getCatalogInfo(int n); + + /* + Add an entry to catalogsinfo database and + @return a refnum for the new added item + */ + + int addCatalogInfo(InfoItem *catInfo,int); + + /* + Search an Item with the same "location" and + @return its refnum. + */ + + int searchCatalogInfo(QString location); + + /* Get a catalog info for location, + * if it doesn't exist it will create one. + * @return the refnum + */ + + int catalogRef(QString location,QString author,QString path); + + /* + Put at refnum the catInfo + @return true if everything is OK + */ + + bool putCatalogInfo(int refnum, InfoItem *catInfo); + + /* + Get word info + */ + WordItem getWordLocations(QString word); + + /* + Add a location for word + */ + bool addLocation(QString word, unsigned int location); + + /* + Remove location for word + */ + bool removeLocation(QString word, int location); + + + /* + * Rebuild location and word indexes + */ + void rebuildIndexes(); + + uint32 appendKey(class QString); + + QString getKey(uint32 n); + + /* + Load the catalogs info. + */ + void loadInfo(); + + + void sync(); + + bool isOk(); + int count(); + int current(); + + void openDataBase(); + void closeDataBase(); + + static QValueList<QString> wordsIn(QString string); + + signals: + + void cannotOpenDB(int); + + + protected: + DataBaseItem cursorGet(uint32 flags); + + QString language; + QString basedir; + QValueList<InfoItem> info; + + DB *db,*infoDb,*wordDb,*indexDb; + DBC *cursor; + bool iAmOk; + bool indexOk; //Database could word without word index +}; + + + + + +#endif diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbscan.cpp b/kbabel/kbabeldict/modules/dbsearchengine/dbscan.cpp new file mode 100644 index 00000000..87b39e61 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbscan.cpp @@ -0,0 +1,197 @@ +/*************************************************************************** + dbscan.cpp - Scan for po files to add in the DB + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + email : rizzi@kde.org + ***************************************************************************/ + +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + + License GPL v 2.0 + + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * + +*/ +#include "dbscan.h" + +#include <qdir.h> +#include <qfile.h> +#include <kapplication.h> +#include <kurl.h> +#include <kdebug.h> + +using namespace KBabel; + +PoScanner::PoScanner(DataBaseManager *dbm, + QObject *parent,const char *name):QObject(parent,name) +{ +dm=dbm; +removeOldCatalogTranslation=false; //Check if this flag do something. +count=0; +} + +bool PoScanner::scanPattern(QString pathName,QString pattern,bool rec) +{ +int tot; + +//Only one progress bar!! +bool pb=false; +static bool called=false; +if (!called) +{ pb=true; count=0;} +called=true; + +kdDebug(0) << QString("cat: %1, %2").arg(pathName).arg(pattern) << endl; + +if(pb) +{emit patternStarted(); +emit patternProgress(0); +} + QDir d(pathName,pattern); + d.setMatchAllDirs(true); + const QFileInfoList* files = d.entryInfoList(); + tot=files->count(); + QPtrListIterator<QFileInfo> it(*files); +kdDebug(0) << tot << endl; + for ( int i=0; i<tot; i++ ) + { + if ((*it)->isDir()) + { + if(rec) + { + kdDebug(0) << d[i] << endl; + if(d[i]!="." && d[i]!="..") + scanPattern(pathName+"/"+d[i],pattern,true); + } + } else + { + kdDebug(0) << d[i] << endl; + scanFile(pathName+"/"+d[i]); + } + + if(pb) + + emit patternProgress(100*i/tot); + + //printf( "%s\n", d[i] ); + + ++it; + } + + + +if(pb) +emit patternProgress(100); + + +if(pb) +emit patternFinished(); +if(pb){called=false;count=0;} + +return true; +} + + + + +bool PoScanner::scanFile(QString fileName) +{ + + +emit fileStarted(); + +InfoItem cinfo; +Catalog * catalog=new Catalog(this,"ScanPoCatalog"); + + +QString location=fileName.right(fileName.length()-fileName.findRev("/")-1); +connect(catalog,SIGNAL(signalProgress(int)),this,SIGNAL(fileLoading(int))); +emit filename(location); +emit fileProgress(0); +emit fileLoading(0); + +KURL u(fileName); + +ConversionStatus rr=catalog->openURL(u); +if(rr != OK && rr !=RECOVERED_PARSE_ERROR ) +{ + delete catalog; + return false; +} +emit fileLoading(100); + +QString author; +if(rr != HEADER_ERROR) + author=catalog->lastTranslator(); +else author=QString("unknown"); + +int catnum=dm->catalogRef(location,author,fileName); + +uint i,tot; +tot=catalog->numberOfEntries(); +//DataBaseItem dbit; +bool fuzzy; +bool untra; + +//kdDebug(0) << QString("Tot: %1").arg(tot) << endl; + +for (i=0;i<tot;i++) //Skip header = ???? +{ + + //Faster ? + if(i % 10==0) + { + emit fileProgress(100*i/tot); + emit added(count); + kapp->processEvents(100); + } + + fuzzy=catalog->isFuzzy(i); + untra=catalog->isUntranslated(i); + + + if(!fuzzy && !untra) + { + int res; + QString msgid,msgstr; + msgid=catalog->msgid(i,true).first(); + kdWarning() << "Translation database does not support plural forms" << endl; + msgstr=catalog->msgstr(i).first(); + res=dm->putNewTranslation(msgid,msgstr,catnum,false); + count+=res; + } + + +} + + +// kdDebug(0) << QString("File finished") << endl; + +emit fileProgress(0); +emit fileLoading(0); +emit fileFinished(); +// dm->loadInfo(); // Sync the list of catalogs NOT NEEDED (?) + +delete catalog; + +//clear(); +return true; + +} +#include "dbscan.moc" diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbscan.h b/kbabel/kbabeldict/modules/dbsearchengine/dbscan.h new file mode 100644 index 00000000..c151509e --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbscan.h @@ -0,0 +1,86 @@ +/*************************************************************************** + dbscan.h - + ------------------- + begin : Fri Sep 8 2000 + copyright : (C) 2000 by Andrea Rizzi + email : rizzi@kde.org + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + * * + * In addition, as a special exception, the copyright holders give * + * permission to link the code of this program with any edition of * + * the Qt library by Trolltech AS, Norway (or with modified versions * + * of Qt that use the same license as Qt), and distribute linked * + * combinations including the two. You must obey the GNU General * + * Public License in all respects for all of the code used other than * + * Qt. If you modify this file, you may extend this exception to * + * your version of the file, but you are not obligated to do so. If * + * you do not wish to do so, delete this exception statement from * + * your version. * + ***************************************************************************/ +/* + Translation search engine + + + Copyright 2000 + Andrea Rizzi rizzi@kde.org + +*/ + + +#ifndef _DBSCAN_H_ +#define _DBSCAN_H_ + +#include <catalog.h> +#include "database.h" + +class PoScanner : public QObject +{ + Q_OBJECT + +public: + + PoScanner(DataBaseManager *dbm,QObject *parent=0,const char *name=0); + + /* + Scan a single PO file. + */ + bool scanFile(QString fileName); + + /* + Scan a list of space separated files with possible MetaCharacters + */ + bool scanPattern(QString pathName,QString pattern="*.po",bool rec=false); + + + + +signals: + void fileStarted(); + void fileProgress(int); + void fileFinished(); + void fileLoading(int); + void patternStarted(); + void patternProgress(int); + void patternFinished(); + void added(int); + void filename(QString); +private: + + // If true when a translation is found in a CATALOG the old translation for this CATALOG + // will be removed + bool removeOldCatalogTranslation; + int count; + DataBaseManager *dm; +// InfoItem cinfo; +}; + + +#endif diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.cpp b/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.cpp new file mode 100644 index 00000000..37332167 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.cpp @@ -0,0 +1,82 @@ + +#include <klocale.h> +#include <kinstance.h> +#include <kaboutdata.h> +#include <kdebug.h> + +#include "dbse_factory.h" +#include "KDBSearchEngine.h" + + +extern "C" +{ + KDE_EXPORT void *init_kbabeldict_dbsearchengine() + { + return new DbSeFactory; + } +} + + +KInstance *DbSeFactory::s_instance = 0; +KAboutData *DbSeFactory::s_about = 0; + + +DbSeFactory::DbSeFactory( QObject *parent, const char *name) + : KLibFactory(parent,name) +{ +} + +DbSeFactory::~DbSeFactory() +{ + if(s_instance) + { + delete s_instance; + s_instance=0; + } + + if(s_about) + { + delete s_about; + s_about=0; + } +} + + +QObject *DbSeFactory::createObject( QObject *parent, const char *name, + const char *classname, const QStringList &) +{ + if(QCString(classname) != "SearchEngine") + { + kdError() << "not a SearchEngine requested" << endl; + return 0; + } + + KDBSearchEngine *se = new KDBSearchEngine(parent,name); + + emit objectCreated(se); + return se; +} + + +KInstance *DbSeFactory::instance() +{ + if(!s_instance) + { + + s_about = new KAboutData( "kdbsearchengine", + I18N_NOOP("Translation Database") + , "0.3" , +I18N_NOOP("A fast translation search engine based on databases") + , KAboutData::License_GPL + , I18N_NOOP("Copyright 2000-2001 by Andrea Rizzi") + ,0,0, "rizzi@kde.org"); + + s_about->addAuthor("Andrea Rizzi",0,"rizzi@kde.org"); + + s_instance = new KInstance(s_about); + } + + return s_instance; +} + +#include "dbse_factory.moc" diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.h b/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.h new file mode 100644 index 00000000..6a9f9f3d --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbse_factory.h @@ -0,0 +1,26 @@ +#ifndef DBSE_FACTORY_H +#define DBSE_FACTORY_H + +#include <klibloader.h> +class KInstance; +class KAboutData; + +class DbSeFactory : public KLibFactory +{ + Q_OBJECT +public: + DbSeFactory( QObject *parent=0, const char *name=0); + ~DbSeFactory(); + + virtual QObject *createObject( QObject *parent=0, const char *name=0, + const char *classname="QObject", + const QStringList &args = QStringList()); + + static KInstance *instance(); + +private: + static KInstance *s_instance; + static KAboutData *s_about; +}; + +#endif diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbsearchengine.desktop b/kbabel/kbabeldict/modules/dbsearchengine/dbsearchengine.desktop new file mode 100644 index 00000000..6aed2e40 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbsearchengine.desktop @@ -0,0 +1,52 @@ +[Desktop Entry] +Type=Service +Name=Translation Database for KBabelDict +Name[bg]=БД с преводи за KBabelDict +Name[br]=Stlennvon geriaoueg evit KBabelDict +Name[bs]=Baza prijevoda za KBabelDict +Name[ca]=Base de dades de traducció per a KBabelDict +Name[cs]=Databáze překladů pro KBabelDict +Name[cy]=Cronfa ddata Cyfieithiadau i KBabelDict +Name[da]=Oversættelsesdatabase for KBabelDict +Name[de]=Übersetzungsdatenbank für KBabelDict +Name[el]=Βάση δεδομένων μετάφρασης για το KBabelDict +Name[es]=Base de datos de traducciones para KBabelDict +Name[et]=KBabelDicti tõlgete andmebaas +Name[eu]=Itzulpen datu-basea KBabelDict-entzat +Name[fa]=دادگان ترجمه برای KBabelDict +Name[fi]=KBabelDict-ohjelman käännöstietokanta +Name[fr]=Base de données des traductions pour KBabelDict +Name[ga]=Cuimhne Aistriúcháin le haghaidh KBabelDict +Name[gl]=Base de datos de traducións de KBabelDict +Name[hi]=के-बेबल-डिक्श के लिए अनुवाद डाटाबेस +Name[hu]=Fordítási adatbázis a KBabelDicthez +Name[is]=Þýðingagagnagrunnur fyrir KBabel orðabókina +Name[it]=Banca dati di traduzioni per KBabelDict +Name[ja]=KBabelDict トランザクションデータベース +Name[ka]=თარგმნის მონაცემთა ბაზა KBabelDict-სთვის +Name[kk]=KBabelDict-тың аударма деректер қоры +Name[lt]=KBabelDict vertimų duomenų bazė +Name[ms]=Pangkalan Data Penterjemahan KBabelDict +Name[nb]=Oversettelsesdatabase for KBabelDict +Name[nds]=Översettendatenbank för KBabelDict +Name[ne]=KBabelDict का लागि अनुबाद डाटाबेस +Name[nl]=Vertalingendatabase voor KBabelDict +Name[nn]=Omsetjingsdatabase for KBabelDict +Name[pa]=ਕੇਬਬੇਲ-ਸ਼ਬਦ-ਕੋਸ਼ ਲਈ ਅਨੁਵਾਦ ਡਾਟਾਬੇਸ +Name[pl]=Baza tłumaczeń dla KBabelDict +Name[pt]=Base de Dados de Traduções do KBabelDict +Name[pt_BR]=Banco de Dados de Traduções para o KBabelDict +Name[ru]=База данных переводов для KBabelDict +Name[sk]=Databáza prekladov pre KBabelDict +Name[sl]=Zbirka prevodov za KBabelDict +Name[sr]=Преводилачка база података за KBabelDict +Name[sr@Latn]=Prevodilačka baza podataka za KBabelDict +Name[sv]=Översättningsdatabas för Kbabeldict +Name[ta]=Kபாபேலுக்கான மொழிபெயர்ப்பு தரவுத்தளம் +Name[tg]=Базаи маълумоти тарҷумаҳо барои KBabelDict +Name[tr]=KBabelDict için Çeviri Veritabanı +Name[uk]=База даних перекладів для KBabelDict +Name[zh_CN]=KBabelDict 的翻译数据库 +Name[zh_TW]=KBabelDict 翻譯資料庫 +X-KDE-Library=kbabeldict_dbsearchengine +ServiceTypes=KBabelDictModule diff --git a/kbabel/kbabeldict/modules/dbsearchengine/dbseprefwidget.ui b/kbabel/kbabeldict/modules/dbsearchengine/dbseprefwidget.ui new file mode 100644 index 00000000..d18ecaef --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/dbseprefwidget.ui @@ -0,0 +1,1036 @@ +<!DOCTYPE UI><UI version="3.0" stdsetdef="1"> +<class>DBSearchEnginePref</class> +<author>Andrea Rizzi <rizzi@kde.org></author> +<widget class="QWidget"> + <property name="name"> + <cstring>DBSEPrefWidget</cstring> + </property> + <property name="geometry"> + <rect> + <x>0</x> + <y>0</y> + <width>414</width> + <height>426</height> + </rect> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QTabWidget" row="0" column="0"> + <property name="name"> + <cstring>TabWidget6</cstring> + </property> + <property name="whatsThis" stdset="0"> + <string></string> + </property> + <widget class="QWidget"> + <property name="name"> + <cstring>Widget4</cstring> + </property> + <attribute name="title"> + <string>Generic</string> + </attribute> + <vbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QButtonGroup"> + <property name="name"> + <cstring>ButtonGroup2</cstring> + </property> + <property name="title"> + <string>Search Mode</string> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QRadioButton" row="0" column="0"> + <property name="name"> + <cstring>allRB</cstring> + </property> + <property name="text"> + <string>Search in whole database (slow)</string> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>Scroll the whole database and return everything that matches +according to the rules defined in tabs <strong> Generic </strong> +and <strong>Match</strong></string> + </property> + </widget> + <widget class="QRadioButton" row="1" column="0"> + <property name="name"> + <cstring>slistRB</cstring> + </property> + <property name="text"> + <string>Search in list of "good keys" (best)</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>Search in a list of <em>good keys</em> (see <strong>Good keys</strong> tab) with rules defined in <strong>Search</strong> tab. +This is the best way to search because the <em>good keys</em> list probably contains all the keys that match with your query. However, it is smaller than the whole database.</string> + </property> + </widget> + <widget class="QRadioButton" row="2" column="0"> + <property name="name"> + <cstring>rlistRB</cstring> + </property> + <property name="text"> + <string>Return the list of "good keys" (fast)</string> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>Returns the whole <em>good keys</em> list. Rules defined in <strong>Search</strong> tab are ignored.</string> + </property> + </widget> + </grid> + </widget> + <widget class="QCheckBox"> + <property name="name"> + <cstring>caseSensitiveCB</cstring> + </property> + <property name="text"> + <string>Case sensitive</string> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>If it is checked the search will be case sensitive. It is ignored if you use <em>Return the list of "good keys"</em> search mode.</string> + </property> + </widget> + <widget class="QCheckBox"> + <property name="name"> + <cstring>normalizeCB</cstring> + </property> + <property name="text"> + <string>Normalize white space</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="whatsThis" stdset="0"> + <string>Remove white spaces at the beginning and at the end of the phrase. +It also substitutes groups of more than one space character with only one space character.</string> + </property> + </widget> + <widget class="QCheckBox"> + <property name="name"> + <cstring>removeContextCB</cstring> + </property> + <property name="text"> + <string>Remove context comment</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="whatsThis" stdset="0"> + <string>Remove, if exists, the _:comment</string> + </property> + </widget> + <widget class="QLayoutWidget"> + <property name="name"> + <cstring>Layout11</cstring> + </property> + <hbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>0</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel"> + <property name="name"> + <cstring>TextLabel3</cstring> + </property> + <property name="text"> + <string>Character to be ignored:</string> + </property> + </widget> + <widget class="KLineEdit"> + <property name="name"> + <cstring>ignoreLE</cstring> + </property> + <property name="sizePolicy"> + <sizepolicy> + <hsizetype>7</hsizetype> + <vsizetype>5</vsizetype> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="minimumSize"> + <size> + <width>0</width> + <height>20</height> + </size> + </property> + </widget> + </hbox> + </widget> + <spacer> + <property name="name"> + <cstring>Spacer1</cstring> + </property> + <property name="orientation"> + <enum>Vertical</enum> + </property> + <property name="sizeType"> + <enum>Expanding</enum> + </property> + </spacer> + </vbox> + </widget> + <widget class="QWidget"> + <property name="name"> + <cstring>Widget5</cstring> + </property> + <attribute name="title"> + <string>Search</string> + </attribute> + <vbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QButtonGroup"> + <property name="name"> + <cstring>ButtonGroup1</cstring> + </property> + <property name="title"> + <string>Matching Method</string> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <spacer row="1" column="0"> + <property name="name"> + <cstring>Spacer4</cstring> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + <property name="sizeType"> + <enum>Fixed</enum> + </property> + <property name="sizeHint"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + <spacer row="2" column="0"> + <property name="name"> + <cstring>Spacer6</cstring> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + <property name="sizeType"> + <enum>Fixed</enum> + </property> + <property name="sizeHint"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + <widget class="QCheckBox" row="2" column="1"> + <property name="name"> + <cstring>containedCB</cstring> + </property> + <property name="text"> + <string>Query is contained</string> + </property> + <property name="whatsThis" stdset="0"> + <string>Match if query is contained in database string</string> + </property> + </widget> + <widget class="QCheckBox" row="3" column="1"> + <property name="name"> + <cstring>containsCB</cstring> + </property> + <property name="text"> + <string>Query contains</string> + </property> + <property name="whatsThis" stdset="0"> + <string>Match if query contains the database string</string> + </property> + </widget> + <widget class="QRadioButton" row="0" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>normalTextRB</cstring> + </property> + <property name="text"> + <string>Normal text</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="whatsThis" stdset="0"> + <string>Consider the search string as normal text.</string> + </property> + </widget> + <widget class="QCheckBox" row="1" column="1"> + <property name="name"> + <cstring>equalCB</cstring> + </property> + <property name="text"> + <string>Equal</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="tristate"> + <bool>false</bool> + </property> + <property name="whatsThis" stdset="0"> + <string>Match if query and database string are equal</string> + </property> + </widget> + <spacer row="3" column="0"> + <property name="name"> + <cstring>Spacer7</cstring> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + <property name="sizeType"> + <enum>Fixed</enum> + </property> + <property name="sizeHint"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + <widget class="QRadioButton" row="4" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>RegExpRB</cstring> + </property> + <property name="text"> + <string>Regular expression</string> + </property> + <property name="whatsThis" stdset="0"> + <string>Consider the search string as a regular expression</string> + </property> + </widget> + </grid> + </widget> + <widget class="QGroupBox"> + <property name="name"> + <cstring>GroupBox3</cstring> + </property> + <property name="title"> + <string>Word Substitution</string> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>If you use one or two <em>word substitution</em> each time you search a phrase with less than the specified number of words, the search engine will also search for all phrases that differ from the original one in one or two words.<p> +<strong>Example:</strong><br> +If you search for <em>My name is Andrea</em> and you have activated <em>one word substitution</em> you may also find phrases like <em>My name is Joe</em> or <em>Your name is Andrea</em>.</string> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <spacer row="3" column="0"> + <property name="name"> + <cstring>Spacer8</cstring> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + <property name="sizeType"> + <enum>Fixed</enum> + </property> + <property name="sizeHint"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + <widget class="QCheckBox" row="0" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>oneWordSubCB</cstring> + </property> + <property name="text"> + <string>Use one word substitution</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="tristate"> + <bool>false</bool> + </property> + </widget> + <spacer row="1" column="0"> + <property name="name"> + <cstring>Spacer9</cstring> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + <property name="sizeType"> + <enum>Fixed</enum> + </property> + <property name="sizeHint"> + <size> + <width>20</width> + <height>20</height> + </size> + </property> + </spacer> + <widget class="QLabel" row="1" column="1"> + <property name="name"> + <cstring>TextLabel1</cstring> + </property> + <property name="text"> + <string>Max number of words in the query:</string> + </property> + </widget> + <widget class="QSpinBox" row="3" column="2"> + <property name="name"> + <cstring>twoWordSubSB</cstring> + </property> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="maxValue"> + <number>14</number> + </property> + <property name="value"> + <number>10</number> + </property> + </widget> + <widget class="QCheckBox" row="2" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>twoWordSubCB</cstring> + </property> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="text"> + <string>Use two word substitution</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + </widget> + <widget class="QLabel" row="3" column="1"> + <property name="name"> + <cstring>TextLabel2</cstring> + </property> + <property name="text"> + <string>Max number of words in the query:</string> + </property> + </widget> + <widget class="QLayoutWidget" row="5" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>Layout7</cstring> + </property> + <hbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>0</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel"> + <property name="name"> + <cstring>TextLabel5_3</cstring> + </property> + <property name="text"> + <string>[A-Za-z0-9_%</string> + </property> + <property name="alignment"> + <set>AlignVCenter|AlignRight</set> + </property> + <property name="hAlign" stdset="0"> + </property> + </widget> + <widget class="KLineEdit"> + <property name="name"> + <cstring>regExpLE</cstring> + </property> + </widget> + <widget class="QLabel"> + <property name="name"> + <cstring>TextLabel6_2</cstring> + </property> + <property name="text"> + <string>]</string> + </property> + </widget> + </hbox> + </widget> + <widget class="QLabel" row="4" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>TextLabel4</cstring> + </property> + <property name="text"> + <string>Local characters for regular expressions:</string> + </property> + </widget> + <widget class="QSpinBox" row="1" column="2"> + <property name="name"> + <cstring>oneWordSubSB</cstring> + </property> + <property name="maxValue"> + <number>200</number> + </property> + <property name="minValue"> + <number>2</number> + </property> + <property name="value"> + <number>40</number> + </property> + </widget> + </grid> + </widget> + <spacer> + <property name="name"> + <cstring>Spacer1_2</cstring> + </property> + <property name="orientation"> + <enum>Vertical</enum> + </property> + <property name="sizeType"> + <enum>Expanding</enum> + </property> + </spacer> + </vbox> + </widget> + <widget class="QWidget"> + <property name="name"> + <cstring>Widget6</cstring> + </property> + <attribute name="title"> + <string>Database</string> + </attribute> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel" row="0" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>TextLabel7_2</cstring> + </property> + <property name="text"> + <string>Database folder:</string> + </property> + </widget> + <widget class="KURLRequester" row="1" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>dirInput</cstring> + </property> + </widget> + <widget class="QCheckBox" row="2" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>autoAddCB_2</cstring> + </property> + <property name="enabled"> + <bool>true</bool> + </property> + <property name="text"> + <string>Auto add entry to database</string> + </property> + <property name="checked"> + <bool>true</bool> + </property> + <property name="whatsThis" stdset="0"> + <string>Automatically add an entry to the database if a new translation is notified by someone (may be kbabel)</string> + </property> + </widget> + <widget class="QLayoutWidget" row="3" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>Layout3</cstring> + </property> + <hbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>0</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel"> + <property name="name"> + <cstring>TextLabel1_4</cstring> + </property> + <property name="text"> + <string>Auto added entry author:</string> + </property> + </widget> + <widget class="QLineEdit"> + <property name="name"> + <cstring>authorLE</cstring> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>Put here the name and email address that you want to use as <em>last translator</em> filed when you auto-add entry to the database (e.g. when you modify a translation with kbabel).<p></string> + </property> + </widget> + </hbox> + </widget> + <widget class="QPushButton" row="4" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>scanFilePB</cstring> + </property> + <property name="text"> + <string>Scan Single PO File...</string> + </property> + </widget> + <widget class="QPushButton" row="5" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>scanPB_2</cstring> + </property> + <property name="text"> + <string>Scan Folder...</string> + </property> + </widget> + <widget class="QPushButton" row="6" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>scanrecPB</cstring> + </property> + <property name="text"> + <string>Scan Folder && Subfolders...</string> + </property> + </widget> + <widget class="QLayoutWidget" row="8" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>Layout5</cstring> + </property> + <vbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>0</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel"> + <property name="name"> + <cstring>filenameLB</cstring> + </property> + <property name="text"> + <string>Scanning file:</string> + </property> + </widget> + <widget class="QLabel"> + <property name="name"> + <cstring>entriesLB</cstring> + </property> + <property name="text"> + <string>Entries added:</string> + </property> + </widget> + </vbox> + </widget> + <widget class="QLayoutWidget" row="9" column="0" rowspan="1" colspan="3"> + <property name="name"> + <cstring>Layout4</cstring> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>0</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QProgressBar" row="2" column="1"> + <property name="name"> + <cstring>processPB</cstring> + </property> + <property name="frameShape"> + <enum>Panel</enum> + </property> + <property name="frameShadow"> + <enum>Sunken</enum> + </property> + <property name="centerIndicator"> + <bool>true</bool> + </property> + <property name="indicatorFollowsStyle"> + <bool>false</bool> + </property> + </widget> + <widget class="QLabel" row="0" column="0"> + <property name="name"> + <cstring>TextLabel1_3</cstring> + </property> + <property name="text"> + <string>Total progress:</string> + </property> + </widget> + <widget class="QLabel" row="2" column="0"> + <property name="name"> + <cstring>TextLabel3_3</cstring> + </property> + <property name="text"> + <string>Processing file:</string> + </property> + </widget> + <widget class="QProgressBar" row="0" column="1"> + <property name="name"> + <cstring>totalPB</cstring> + </property> + <property name="frameShape"> + <enum>Panel</enum> + </property> + <property name="frameShadow"> + <enum>Sunken</enum> + </property> + <property name="centerIndicator"> + <bool>true</bool> + </property> + </widget> + <widget class="QProgressBar" row="1" column="1"> + <property name="name"> + <cstring>loadingPB</cstring> + </property> + <property name="frameShape"> + <enum>Panel</enum> + </property> + <property name="frameShadow"> + <enum>Sunken</enum> + </property> + <property name="centerIndicator"> + <bool>true</bool> + </property> + </widget> + <widget class="QLabel" row="1" column="0"> + <property name="name"> + <cstring>TextLabel2_3</cstring> + </property> + <property name="text"> + <string>Loading file:</string> + </property> + </widget> + </grid> + </widget> + <widget class="QPushButton" row="10" column="2"> + <property name="name"> + <cstring>exportPB</cstring> + </property> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="text"> + <string>Export...</string> + </property> + </widget> + <widget class="QPushButton" row="10" column="0"> + <property name="name"> + <cstring>statPB</cstring> + </property> + <property name="enabled"> + <bool>false</bool> + </property> + <property name="text"> + <string>Statistics</string> + </property> + </widget> + <widget class="QPushButton" row="10" column="1"> + <property name="name"> + <cstring>repeatPB</cstring> + </property> + <property name="text"> + <string>Repeated Strings</string> + </property> + </widget> + </grid> + </widget> + <widget class="QWidget"> + <property name="name"> + <cstring>tab</cstring> + </property> + <attribute name="title"> + <string>Good Keys</string> + </attribute> + <vbox> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QGroupBox"> + <property name="name"> + <cstring>GroupBox4</cstring> + </property> + <property name="title"> + <string>Generic</string> + </property> + <property name="whatsThis" stdset="0"> + <string><qml>Here you can define how to fill the <em>good keys list</em>.<p> +You can set the minimum number of words of the query that a key must have to be inserted in the <em>good keys list</em>.<p> +You can also set the minimum number of words of the key that the query must have to insert the key in the list.<p> +These two numbers are the percentage of the total number of words. If the result of this percentage is less than one, the engine will set it to one.<p> +Finally you can set the maximum number of entries in the list.</string> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel" row="2" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>TextLabel3_2</cstring> + </property> + <property name="text"> + <string>Minimum number of words of the key also in the query (%):</string> + </property> + <property name="textFormat"> + <enum>RichText</enum> + </property> + </widget> + <widget class="QSlider" row="1" column="0"> + <property name="name"> + <cstring>thresholdSL</cstring> + </property> + <property name="maxValue"> + <number>100</number> + </property> + <property name="value"> + <number>50</number> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + </widget> + <widget class="QSpinBox" row="1" column="1"> + <property name="name"> + <cstring>SpinBox5</cstring> + </property> + <property name="suffix"> + <string>%</string> + </property> + <property name="maxValue"> + <number>100</number> + </property> + <property name="value"> + <number>50</number> + </property> + </widget> + <widget class="QLabel" row="0" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>TextLabel2_2</cstring> + </property> + <property name="text"> + <string>Minimum number of query words in the key (%):</string> + </property> + </widget> + <widget class="QSpinBox" row="4" column="1"> + <property name="name"> + <cstring>maxSB</cstring> + </property> + <property name="maxValue"> + <number>5000</number> + </property> + <property name="value"> + <number>30</number> + </property> + </widget> + <widget class="QSpinBox" row="3" column="1"> + <property name="name"> + <cstring>SpinBox6</cstring> + </property> + <property name="suffix"> + <string>%</string> + </property> + <property name="maxValue"> + <number>100</number> + </property> + <property name="value"> + <number>50</number> + </property> + </widget> + <widget class="QLabel" row="4" column="0"> + <property name="name"> + <cstring>TextLabel4_2</cstring> + </property> + <property name="text"> + <string>Max list length:</string> + </property> + </widget> + <widget class="QSlider" row="3" column="0"> + <property name="name"> + <cstring>thresholdOrigSL</cstring> + </property> + <property name="maxValue"> + <number>100</number> + </property> + <property name="value"> + <number>50</number> + </property> + <property name="orientation"> + <enum>Horizontal</enum> + </property> + </widget> + </grid> + </widget> + <widget class="QGroupBox"> + <property name="name"> + <cstring>GroupBox3_2</cstring> + </property> + <property name="title"> + <string>Frequent Words</string> + </property> + <grid> + <property name="name"> + <cstring>unnamed</cstring> + </property> + <property name="margin"> + <number>11</number> + </property> + <property name="spacing"> + <number>6</number> + </property> + <widget class="QLabel" row="0" column="0"> + <property name="name"> + <cstring>TextLabel1_2</cstring> + </property> + <property name="text"> + <string>Discard words more frequent than:</string> + </property> + </widget> + <widget class="QSpinBox" row="0" column="1"> + <property name="name"> + <cstring>freqSB</cstring> + </property> + <property name="suffix"> + <string>/10000</string> + </property> + <property name="maxValue"> + <number>10000</number> + </property> + <property name="lineStep"> + <number>1</number> + </property> + <property name="value"> + <number>100</number> + </property> + </widget> + <widget class="QCheckBox" row="1" column="0" rowspan="1" colspan="2"> + <property name="name"> + <cstring>nothingCB</cstring> + </property> + <property name="text"> + <string>Frequent words are considered as in every key</string> + </property> + </widget> + </grid> + </widget> + <spacer> + <property name="name"> + <cstring>Spacer3</cstring> + </property> + <property name="orientation"> + <enum>Vertical</enum> + </property> + <property name="sizeType"> + <enum>Expanding</enum> + </property> + </spacer> + </vbox> + </widget> + </widget> + </grid> +</widget> +<connections> + <connection> + <sender>thresholdSL</sender> + <signal>valueChanged(int)</signal> + <receiver>SpinBox5</receiver> + <slot>setValue(int)</slot> + </connection> + <connection> + <sender>thresholdOrigSL</sender> + <signal>valueChanged(int)</signal> + <receiver>SpinBox6</receiver> + <slot>setValue(int)</slot> + </connection> + <connection> + <sender>SpinBox5</sender> + <signal>valueChanged(int)</signal> + <receiver>thresholdSL</receiver> + <slot>setValue(int)</slot> + </connection> + <connection> + <sender>SpinBox6</sender> + <signal>valueChanged(int)</signal> + <receiver>thresholdOrigSL</receiver> + <slot>setValue(int)</slot> + </connection> +</connections> +<includes> + <include location="local" impldecl="in declaration">klocale.h</include> + <include location="global" impldecl="in declaration">kseparator.h</include> +</includes> +<layoutdefaults spacing="6" margin="11"/> +</UI> diff --git a/kbabel/kbabeldict/modules/dbsearchengine/makemsgdb.C b/kbabel/kbabeldict/modules/dbsearchengine/makemsgdb.C new file mode 100644 index 00000000..a83d947b --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/makemsgdb.C @@ -0,0 +1,327 @@ +#include <stdio.h> +#include <time.h> +#include <errno.h> +#include <gdbm.h> +#include <string.h> + +void removechar (char *s, int c) +{ + int i, l; + l = strlen (s); + if ((c >= l) || (c < 0)) + return; + for (i = c; i < l; i++) + s[i] = s[i + 1]; +} +void removeallc (char *s, char c) +{ + char *pos; + while ((pos = strchr (s, c)) != 0) + removechar (s, (long int) pos - (long int) s); + +} + +void normalizestr (char *s) +{ + char *pos; + while ((pos = strstr (s, "#~")) != 0) { + removechar (s, (long int) pos - (long int) s); + removechar (s, (long int) pos - (long int) s); + } + while (strchr (s, ' ') == s) + removechar (s, 0); +} + +void freadline(char *buff,FILE *f) +{ +char c; +while ((fread(&c,1,1,f)==1) && (c!='\n')) + { + *buff=c; + buff++; + } +*buff=0; + +} + +int makePoDb(const char* sourceName,const char* outputName) +{ +static bool open = false; +int m=0,n=0,h=0; +GDBM_FILE db; +datum key,value; +char *s,a[20000],b[2000],k[2000],v[2000]; +int i,*np,nmax=0,co=-1,oldref[2000]; +long int tim; +FILE *mlf; +bool nextIsFuzzy; +bool isAMsgId=true; +/* char keystring[1000],valuestring[1000]; */ + +/*Read headers, refnum end other info */ +db = gdbm_open((char *)outputName,1024,GDBM_READER,0666,0); +mlf = fopen(sourceName,"r"); +if(strrchr(sourceName,'/')!=0) +sourceName=strrchr(sourceName,'/')+1; + +if(!(db==0)) + { + printf("ciao\n"); + key.dptr=a; + strcpy(a,"__@REFNUM__"); + key.dsize=strlen(a)+1; + value = gdbm_fetch(db,key); + np=(int*)value.dptr; + nmax=*np; + for(i=0;i<nmax;i++) { + sprintf(a,"__@%d__",i); + key.dsize=strlen(a)+1; + value = gdbm_fetch(db,key); + if(strcmp(value.dptr,sourceName)==0) + oldref[++co]=i; + // printf("File %s in Date %s\nRef # %d, oldref[%d]=%d\n",value.dptr,value.dptr+1+strlen(value.dptr),i,co,oldref[co]); + } + gdbm_close(db); + } + + +db = gdbm_open((char *)outputName,1024,GDBM_WRCREAT,0666,0); +if(db==0) return 0; + +nmax++; +//sourceName=strrchr(sourceName,'/')+1; + if(open) return 0; + open=true; + + key.dptr=a; + strcpy(a,"__@REFNUM__"); + key.dsize=strlen(a)+1; + value.dptr=(char *)&nmax; + value.dsize=4; + gdbm_store(db,key,value,GDBM_REPLACE); + sprintf(a,"__@%d__",nmax-1); + key.dsize=strlen(a)+1; + strcpy(v,sourceName); + value.dptr=v; + time(&tim); +// ctime(&tim); + // fprintf(stderr,"%s %s\n",v,ctime(&tim)); + strcpy(v+strlen(v)+1,ctime(&tim)); + value.dsize=strlen(v)+2+strlen(v+strlen(v)+1); + gdbm_store(db,key,value,GDBM_REPLACE); + + while(!feof(mlf)) + { + freadline(a,mlf); + normalizestr(a); +// printf("#%s#\n",a); + +// while (st.find("#~")==0) +// st = st.right(st.length()-2); +// while (st.find(" ")==0) +// st = st.right(st.length()-1); + + if(isAMsgId) nextIsFuzzy=false; + if(strstr(a,"#,")==a) + if(strstr(a,"fuzzy")!=0) nextIsFuzzy=true; + isAMsgId=(strstr(a,"msgid")); + if(isAMsgId && !nextIsFuzzy) + { + *b='\0'; + clearerr(mlf); + while(!feof(mlf) && !(strstr(a,"msgstr")==a)) + { + strcat(b,strchr(a,'"')+1); + *strrchr(b,'\"')= '\0'; //" + freadline(a,mlf); + normalizestr(a); + + + if(b+strlen(b)==strstr(b,"\\n")+2) + { + b[strlen(b)-2]='\n'; + b[strlen(b)-1]='\0'; + } + } + + // fprintf(stderr,"MSGID#%s#\n",b); + //} + + if(b[0]!='\0') { + key.dptr=k; + strcpy(k,b); + key.dsize=strlen(key.dptr)+1; + int lines=0; + *b='\0'; + while(!feof(mlf) && !(strstr(a,"msgid")==a) && !(strchr(a,'"')==0)) + { + lines++; + strcat(b,strchr(a,'"')+1); + *strrchr(b,'\"')= '\0'; //" + freadline(a,mlf); + normalizestr(a); + + + if(b+strlen(b)==strstr(b,"\\n")+2) + { + b[strlen(b)-2]='\n'; + b[strlen(b)-1]='\0'; + } + } + value.dptr=v; + int *t; + int *nr,*re; //,*md; //Number of references,ref + + t=(int *)value.dptr; + *t=1; + strcpy(value.dptr+4,b); + nr=(int *)(value.dptr+4+strlen(value.dptr+4)+1); + *nr=1; + nr++; + *nr=nmax-1; + value.dsize=strlen(value.dptr+4)+1+4+8; + // fprintf(stderr,"MSGSTR#%s#nref=%d\n",value.dptr+4, +// *(int*)(value.dptr+4+strlen(value.dptr+4)+1 )); + + if(b[0]!='\0') { + if(gdbm_store(db,key,value,GDBM_INSERT)) + { + // fprintf(stderr,"*** Key exists ***\n"); + + gdbm_sync(db); + gdbm_close(db); + db = gdbm_open((char *)outputName,1024,GDBM_READER,0,0); + datum oldvalue; + oldvalue= gdbm_fetch(db,key); + gdbm_sync(db); + gdbm_close(db); + db = gdbm_open((char *)outputName,1024,GDBM_WRCREAT,0,0); + + t=(int *)oldvalue.dptr; + int i,r,j; //counters + int v=*t; //Number of strings + int *nr,*re,*md; //Number of references,ref + bool exist=false,here,modif=false; + char *os=oldvalue.dptr+4; + // fprintf(stderr,"**Searching string #%s#\n",b); + for(i=0;i<v;i++) + { + + exist=false; //REMOVE THIS LINE!!! + here=false; + // fprintf(stderr,"**STRING %d #%s# len=%d %s\n",i,os,strlen(os),b); + if(strcmp(os,b)==0) { + exist=true; + here=true; + // fprintf(stderr,"That's good\n"); + } + os+=strlen(os)+1; //End of string + nr=(int *)os; + os+=(*nr+1)*4; + re=nr; + modif=false; + // fprintf(stderr,"refernces %d\n",*nr); + for(j=0;j<*nr;j++) + { + re++; + + if(here) + { + // printf("reference #%d\n",*re); + for(r=0;r<=co;r++) + { + // fprintf(stderr,"%d==%d ?-->",oldref[co],*re); + if(oldref[r]==*re) + { + modif=true; + // fprintf(stderr,"Yes\n"); + *re=(nmax-1); + } //else fprintf(stderr,"No\n"); + } + // fprintf(stderr,"qui\n"); + if(!modif) + md=nr; + // fprintf(stderr,"modif %s\n",modif ? "true":"false"); + } + } + + } + + if(!exist) + { + int oldlen=(long int)os-(long int)oldvalue.dptr-4; + memcpy(a+4,oldvalue.dptr+4,oldlen); +// fprintf(stderr,"***!exist Old len is %d+4 1st str is %s\n",oldlen,a+4); + v++; + t=(int *)a; + *t=v; + // fprintf(stderr,"b=%s",b); + strcpy(a+4+oldlen,b); + re=(int *)(a+4+oldlen+strlen(b)+1); + *re=1; + re++; + *re=nmax-1; + //fprintf(stderr,"a+4=%s a+4+oldlen=%s",a+4,a+4+oldlen); + value.dptr=a; + value.dsize=oldlen+strlen(b)+1+4+8; + gdbm_store(db,key,value,GDBM_REPLACE); + n++; + } else + { + if(!modif) + { +// fprintf(stderr,"grossa crisi %d\n",*md); +// fprintf(stderr,"Old num of ref \n"); + int oldlen1=(long int)(md)-(long int)(oldvalue.dptr)-4; + int oldlen2=(long int)(os)-(long int)(md)-4; //-4 because nr + memcpy(a+4,oldvalue.dptr+4,oldlen1); + memcpy(a+4+oldlen1+8,oldvalue.dptr+4+oldlen1+4,oldlen2); + re=(int *)(a+4+oldlen1); + *re=(*md )+1; + // *re++; + re++; + *re=nmax-1; + t=(int *)a; + *t=v; + value.dptr=a; + value.dsize=oldlen1+oldlen2+4+8; + gdbm_store(db,key,value,GDBM_REPLACE); + n++; + } + else //if (modif) + { + value.dptr=oldvalue.dptr; + value.dsize=oldvalue.dsize; + gdbm_store(db,key,value,GDBM_REPLACE); + } + } + + h++; + } else { + + m++; + } + } + } + + } + + + } + + fclose(mlf); + gdbm_close(db); + open=false; + printf("new Key in database %d\n old key found %d\n value added %d\n",m,h,n); + return m+n; +} + + + +main(int argc,char **argv) +{ +int i; +for(i=1;i<argc;i++) + printf("File %s:\nEntry added to dbase: %d\n",argv[i],makePoDb(argv[i],"messages2.db")); + +} diff --git a/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.cpp b/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.cpp new file mode 100644 index 00000000..3bb65934 --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.cpp @@ -0,0 +1,111 @@ +#include <qradiobutton.h> +#include <qslider.h> +#include <qspinbox.h> +#include <qcheckbox.h> +#include <qlabel.h> +#include <qlayout.h> +#include <klocale.h> +#include <kfiledialog.h> +#include <kurlrequester.h> +#include <qtoolbutton.h> +#include <klineedit.h> +#include <kstandarddirs.h> + +#include "dbseprefwidget.h" +#include "preferenceswidget.h" + +PreferencesWidget::PreferencesWidget(QWidget *parent, const char* name) + : PrefWidget(parent,name) +{ + QVBoxLayout *layout = new QVBoxLayout(this); +// QLabel *label = new QLabel(i18n("Settings for KDE database search engine"),this); +// layout->addWidget(label); + + dbpw = new DBSearchEnginePref(this); + dbpw->dirInput->setMode(KFile::Directory | KFile::LocalOnly); + + layout->addWidget(dbpw); + resize(QSize(200,200).expandedTo(minimumSizeHint())); + +// connect(dbpw->browseTB_3,SIGNAL(clicked()),SLOT(browse1())); + + emit restoreNow(); //Fill with actual params. + +} + +PreferencesWidget::~PreferencesWidget() +{ +} + +void PreferencesWidget::apply() +{ +emit applyNow(); +} + +void PreferencesWidget::cancel() +{ +emit restoreNow(); +} + +void PreferencesWidget::standard() +{ + + +dbpw->caseSensitiveCB->setChecked(false); +dbpw->normalizeCB->setChecked(true); +dbpw->removeContextCB->setChecked(true); + +dbpw->oneWordSubCB->setChecked(true); +dbpw->twoWordSubCB->setChecked(false); + + +dbpw->RegExpRB->setChecked(false); +dbpw->normalTextRB->setChecked(true); +dbpw->equalCB->setChecked( true ); +dbpw->containsCB->setChecked( true); +dbpw->containedCB->setChecked( true ); + + +dbpw->oneWordSubSB->setValue(20); +dbpw->twoWordSubSB->setValue(8); + +dbpw->maxSB->setValue(500); +dbpw->thresholdSL->setValue(50); +dbpw->thresholdOrigSL->setValue(50); + +dbpw->allRB->setChecked( false); +dbpw->slistRB->setChecked( true); +dbpw->rlistRB->setChecked( false ); + +dbpw->nothingCB->setChecked(true); +dbpw->freqSB->setValue(300); + +dbpw->ignoreLE->setText("&.:"); + +dbpw->autoAddCB_2->setChecked(true); + +QString defaultDir; + KStandardDirs * dirs = KGlobal::dirs(); + if(dirs) + { + defaultDir = dirs->saveLocation("data"); + if(defaultDir.right(1)!="/") + defaultDir+="/"; + defaultDir += "kbabeldict/dbsearchengine"; + } + + dbpw->dirInput->setURL(defaultDir); +} + +void PreferencesWidget::setName(QString n) +{ +dbpw->filenameLB->setText(i18n("Scanning file: %1").arg(n)); +} + +void PreferencesWidget::setEntries(int i) +{ +dbpw->entriesLB->setText(i18n("Entries added: %1").arg(i)); + +} + +#include "preferenceswidget.moc" diff --git a/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.h b/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.h new file mode 100644 index 00000000..599408cd --- /dev/null +++ b/kbabel/kbabeldict/modules/dbsearchengine/preferenceswidget.h @@ -0,0 +1,28 @@ +#ifndef PREFERENCESWIDGET_H +#define PREFERENCESWIDGET_H + +#include "searchengine.h" +#include "dbseprefwidget.h" + +class PreferencesWidget : public PrefWidget +{ + Q_OBJECT + +public: + PreferencesWidget(QWidget *parent=0, const char* name=0); + virtual ~PreferencesWidget(); + + virtual void apply(); + virtual void cancel(); + virtual void standard(); + DBSearchEnginePref *dbpw; +public slots: + void setName(QString); + void setEntries(int); +signals: + void applyNow(); + void restoreNow(); + +}; + +#endif |