data/FolderEvidenceModel.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #include "FolderEvidenceModel.h" | ||
| 2 | |||
| 3 | #include "MailCacheSql.h" | ||
| 4 | #include "Models.h" | ||
| 5 | |||
| 6 | #include <QElapsedTimer> | ||
| 7 | #include <QLoggingCategory> | ||
| 8 | #include <QMutexLocker> | ||
| 9 | #include <QSqlError> | ||
| 10 | #include <QSqlQuery> | ||
| 11 | #include <QTimeZone> | ||
| 12 | |||
| 13 |
4/7✓ Branch 2 → 3 taken 13 times.
✓ Branch 2 → 8 taken 515 times.
✓ Branch 4 → 5 taken 13 times.
✗ Branch 4 → 8 not taken.
✓ Branch 5 → 6 taken 13 times.
✗ Branch 10 → 11 not taken.
✗ Branch 10 → 12 not taken.
|
528 | Q_LOGGING_CATEGORY(lcFolderEvidence, "mailjd.predictor.evidence") |
| 14 | |||
| 15 | 15953 | const QStringList &FolderEvidenceModel::excludedFolders() { | |
| 16 |
5/10✓ Branch 2 → 3 taken 22 times.
✓ Branch 2 → 19 taken 15931 times.
✓ Branch 4 → 5 taken 22 times.
✗ Branch 4 → 19 not taken.
✓ Branch 16 → 17 taken 154 times.
✓ Branch 16 → 18 taken 22 times.
✗ Branch 22 → 23 not taken.
✗ Branch 22 → 24 not taken.
✗ Branch 31 → 32 not taken.
✗ Branch 31 → 33 not taken.
|
16129 | static const QStringList folders = {"INBOX", "Sent", "Trash", "Drafts", "Archive", "Junk", "Spam"}; |
| 17 | 15953 | return folders; | |
| 18 | 22 | } | |
| 19 | |||
| 20 | 15221 | bool FolderEvidenceModel::excludedFolder(const QString &path) { | |
| 21 |
3/3✓ Branch 2 → 3 taken 15221 times.
✓ Branch 42 → 5 taken 25158 times.
✓ Branch 42 → 43 taken 927 times.
|
26085 | for (const QString &folder : excludedFolders()) |
| 22 |
5/8✓ Branch 10 → 11 taken 10866 times.
✓ Branch 11 → 12 taken 10866 times.
✓ Branch 12 → 13 taken 10866 times.
✗ Branch 12 → 28 not taken.
✓ Branch 38 → 39 taken 14294 times.
✓ Branch 38 → 40 taken 10864 times.
✗ Branch 63 → 64 not taken.
✗ Branch 63 → 65 not taken.
|
36024 | if (path.compare(folder, Qt::CaseInsensitive) == 0 || path.startsWith(folder + QLatin1Char('.'), Qt::CaseInsensitive) |
| 23 |
6/8✓ Branch 15 → 16 taken 10866 times.
✓ Branch 16 → 17 taken 10866 times.
✓ Branch 17 → 18 taken 10865 times.
✓ Branch 17 → 28 taken 1 time.
✓ Branch 36 → 37 taken 10866 times.
✓ Branch 36 → 38 taken 14292 times.
✗ Branch 57 → 58 not taken.
✗ Branch 57 → 59 not taken.
|
36024 | || path.startsWith(folder + QLatin1Char('/'), Qt::CaseInsensitive) |
| 24 |
6/8✓ Branch 20 → 21 taken 10865 times.
✓ Branch 21 → 22 taken 10865 times.
✓ Branch 22 → 23 taken 10864 times.
✓ Branch 22 → 28 taken 1 time.
✓ Branch 34 → 35 taken 10866 times.
✓ Branch 34 → 36 taken 14292 times.
✗ Branch 51 → 52 not taken.
✗ Branch 51 → 53 not taken.
|
36023 | || path.endsWith(QLatin1Char('.') + folder, Qt::CaseInsensitive) |
| 25 |
9/12✓ Branch 7 → 8 taken 10866 times.
✓ Branch 7 → 28 taken 14292 times.
✓ Branch 25 → 26 taken 10864 times.
✓ Branch 26 → 27 taken 10864 times.
✗ Branch 27 → 28 not taken.
✓ Branch 27 → 29 taken 10864 times.
✓ Branch 30 → 31 taken 10864 times.
✓ Branch 30 → 32 taken 14294 times.
✓ Branch 32 → 33 taken 10865 times.
✓ Branch 32 → 34 taken 14293 times.
✗ Branch 45 → 46 not taken.
✗ Branch 45 → 47 not taken.
|
36024 | || path.endsWith(QLatin1Char('/') + folder, Qt::CaseInsensitive)) |
| 26 | 14294 | return true; | |
| 27 | 927 | return false; | |
| 28 | } | ||
| 29 | |||
| 30 | 903 | bool FolderEvidenceModel::built(const QString &account) const { | |
| 31 | 903 | QMutexLocker locker(&m_mutex); | |
| 32 |
3/4✓ Branch 3 → 4 taken 375 times.
✓ Branch 3 → 7 taken 528 times.
✓ Branch 5 → 6 taken 375 times.
✗ Branch 5 → 7 not taken.
|
903 | return m_stats.revision > 0 && m_account == account; |
| 33 | 903 | } | |
| 34 | |||
| 35 | 433 | bool FolderEvidenceModel::learnRows(const QSqlDatabase &db, qint64 afterId, bool replay, QString *error) { | |
| 36 | // Bounded fields (8 KiB like the predictor's reader); date order. | ||
| 37 |
1/1✓ Branch 2 → 3 taken 433 times.
|
433 | QSqlQuery query(db); |
| 38 |
1/1✓ Branch 3 → 4 taken 433 times.
|
433 | query.setForwardOnly(true); |
| 39 |
1/1✓ Branch 7 → 8 taken 433 times.
|
433 | query.prepare(QStringLiteral( |
| 40 | "SELECT h.id, f.path, substr(h.from_addr,1,8192), substr(h.to_addr,1,8192), substr(h.cc_addr,1,8192), " | ||
| 41 | "substr(h.subject,1,8192), substr(h.list_id,1,1024), substr(h.message_id,1,1024), substr(h.in_reply_to,1,8192), " | ||
| 42 | "substr(h.ref_ids,1,8192), h.date FROM headers h JOIN folders f ON f.id = h.folder_id " | ||
| 43 | "WHERE f.account = ? AND h.id > ? ORDER BY h.date, h.id")); | ||
| 44 |
1/1✓ Branch 12 → 13 taken 433 times.
|
433 | query.addBindValue(m_account); |
| 45 |
1/1✓ Branch 16 → 17 taken 433 times.
|
433 | query.addBindValue(afterId); |
| 46 |
2/3✓ Branch 18 → 19 taken 433 times.
✗ Branch 19 → 20 not taken.
✓ Branch 19 → 28 taken 433 times.
|
433 | if (!query.exec()) { |
| 47 | ✗ | if (error) *error = query.lastError().text(); | |
| 48 | ✗ | return false; | |
| 49 | } | ||
| 50 | 433 | QList<FolderReplayItem> items; | |
| 51 | 433 | qint64 last = afterId; | |
| 52 |
3/3✓ Branch 108 → 109 taken 15598 times.
✓ Branch 109 → 29 taken 15165 times.
✓ Branch 109 → 110 taken 433 times.
|
15598 | while (query.next()) { |
| 53 |
2/2✓ Branch 29 → 30 taken 15165 times.
✓ Branch 30 → 31 taken 15165 times.
|
15165 | last = std::max(last, query.value(0).toLongLong()); |
| 54 |
2/2✓ Branch 33 → 34 taken 15165 times.
✓ Branch 34 → 35 taken 15165 times.
|
15165 | const QString path = query.value(1).toString(); |
| 55 |
3/3✓ Branch 36 → 37 taken 15165 times.
✓ Branch 37 → 38 taken 14276 times.
✓ Branch 37 → 39 taken 889 times.
|
15165 | if (excludedFolder(path)) continue; |
| 56 | 889 | MailHeader header; | |
| 57 |
2/2✓ Branch 40 → 41 taken 889 times.
✓ Branch 41 → 42 taken 889 times.
|
889 | header.from = query.value(2).toString(); |
| 58 |
2/2✓ Branch 45 → 46 taken 889 times.
✓ Branch 46 → 47 taken 889 times.
|
889 | header.to = query.value(3).toString(); |
| 59 |
2/2✓ Branch 50 → 51 taken 889 times.
✓ Branch 51 → 52 taken 889 times.
|
889 | header.cc = query.value(4).toString(); |
| 60 |
2/2✓ Branch 55 → 56 taken 889 times.
✓ Branch 56 → 57 taken 889 times.
|
889 | header.subject = query.value(5).toString(); |
| 61 |
2/2✓ Branch 60 → 61 taken 889 times.
✓ Branch 61 → 62 taken 889 times.
|
889 | header.listId = query.value(6).toString(); |
| 62 |
2/2✓ Branch 65 → 66 taken 889 times.
✓ Branch 66 → 67 taken 889 times.
|
889 | header.messageId = query.value(7).toString(); |
| 63 |
3/3✓ Branch 70 → 71 taken 889 times.
✓ Branch 71 → 72 taken 889 times.
✓ Branch 72 → 73 taken 889 times.
|
889 | header.inReplyToIds = MailCacheSql::decodeThreadIds(query.value(8).toString()); |
| 64 |
3/3✓ Branch 77 → 78 taken 889 times.
✓ Branch 78 → 79 taken 889 times.
✓ Branch 79 → 80 taken 889 times.
|
889 | header.references = MailCacheSql::decodeThreadIds(query.value(9).toString()); |
| 65 |
7/8✓ Branch 84 → 85 taken 889 times.
✓ Branch 85 → 86 taken 889 times.
✓ Branch 87 → 88 taken 889 times.
✗ Branch 87 → 97 not taken.
✓ Branch 88 → 89 taken 889 times.
✓ Branch 89 → 90 taken 889 times.
✓ Branch 90 → 91 taken 889 times.
✓ Branch 91 → 92 taken 889 times.
|
889 | if (!query.value(10).isNull()) header.date = QDateTime::fromSecsSinceEpoch(query.value(10).toLongLong(), QTimeZone::utc()); |
| 66 | 889 | items.append({FolderEvidence::fromHeader(header), path}); | |
| 67 | 15165 | } | |
| 68 |
2/2✓ Branch 110 → 111 taken 264 times.
✓ Branch 110 → 154 taken 169 times.
|
433 | if (replay) { |
| 69 | // The replay trains and calibrates in one pass (out of sample). | ||
| 70 | 264 | m_evidence = FolderEvidence(); | |
| 71 | 264 | m_calibration = FolderCalibration(); | |
| 72 |
1/1✓ Branch 117 → 118 taken 264 times.
|
264 | const FolderReplayMetrics metrics = replayFolderEvidence(items, 0.0, &m_calibration, &m_evidence); |
| 73 | // A threshold measured on a handful of filings says nothing; below | ||
| 74 | // MinReplayAnswers the long-standing 0.4 of the word model applies. | ||
| 75 |
6/8✓ Branch 118 → 119 taken 1 time.
✓ Branch 118 → 121 taken 263 times.
✓ Branch 119 → 120 taken 1 time.
✓ Branch 121 → 122 taken 263 times.
✓ Branch 123 → 124 taken 263 times.
✓ Branch 123 → 125 taken 1 time.
✗ Branch 252 → 253 not taken.
✗ Branch 252 → 254 not taken.
|
264 | m_stats.threshold = metrics.answered >= MinReplayAnswers ? metrics.thresholdFor(0.85) : FolderSuggestion().threshold; |
| 76 |
1/1✓ Branch 125 → 126 taken 264 times.
|
264 | const auto [coverage, precision] = metrics.at(m_stats.threshold); |
| 77 | 264 | m_stats.replayCoverage = coverage; | |
| 78 | 264 | m_stats.replayPrecision = precision; | |
| 79 |
8/8✓ Branch 128 → 129 taken 264 times.
✓ Branch 132 → 133 taken 264 times.
✓ Branch 138 → 139 taken 264 times.
✓ Branch 139 → 140 taken 264 times.
✓ Branch 141 → 142 taken 264 times.
✓ Branch 142 → 143 taken 264 times.
✓ Branch 151 → 130 taken 264 times.
✓ Branch 151 → 152 taken 264 times.
|
1056 | qCInfo(lcFolderEvidence).noquote() << QStringLiteral("replay %1; threshold %2").arg(metrics.report()).arg(m_stats.threshold); |
| 80 | 264 | } else { | |
| 81 |
1/3✗ Branch 158 → 159 not taken.
✗ Branch 161 → 157 not taken.
✓ Branch 161 → 162 taken 169 times.
|
169 | for (const FolderReplayItem &item : std::as_const(items)) m_evidence.learn(item.mail, item.folder); |
| 82 | } | ||
| 83 | 433 | m_lastHeaderId = last; | |
| 84 | 433 | m_stats.filings = m_evidence.filings(); | |
| 85 | 433 | return true; | |
| 86 | 1322 | } | |
| 87 | |||
| 88 | 264 | bool FolderEvidenceModel::build(const QSqlDatabase &db, const QString &account, QString *error) { | |
| 89 | 264 | QMutexLocker locker(&m_mutex); | |
| 90 | 264 | QElapsedTimer timer; | |
| 91 | 264 | timer.start(); | |
| 92 | 264 | m_account = account; | |
| 93 |
2/3✓ Branch 5 → 6 taken 264 times.
✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 264 times.
|
264 | if (!learnRows(db, 0, true, error)) return false; |
| 94 | 264 | m_stats.buildMs = timer.elapsed(); | |
| 95 | 264 | ++m_stats.revision; | |
| 96 |
9/9✓ Branch 9 → 10 taken 264 times.
✓ Branch 13 → 14 taken 264 times.
✓ Branch 14 → 15 taken 264 times.
✓ Branch 15 → 16 taken 264 times.
✓ Branch 16 → 17 taken 264 times.
✓ Branch 17 → 18 taken 264 times.
✓ Branch 18 → 19 taken 264 times.
✓ Branch 22 → 11 taken 264 times.
✓ Branch 22 → 23 taken 264 times.
|
528 | qCInfo(lcFolderEvidence) << "evidence built:" << m_stats.filings << "filings in" << m_stats.buildMs << "ms"; |
| 97 | 264 | return true; | |
| 98 | 264 | } | |
| 99 | |||
| 100 | 169 | bool FolderEvidenceModel::update(const QSqlDatabase &db, QString *error) { | |
| 101 | 169 | QMutexLocker locker(&m_mutex); | |
| 102 |
1/1✓ Branch 3 → 4 taken 169 times.
|
338 | return learnRows(db, m_lastHeaderId, false, error); |
| 103 | 169 | } | |
| 104 | |||
| 105 | 37 | void FolderEvidenceModel::correct(const MailHeader &header, const QString &chosenFolder) { | |
| 106 | 37 | QMutexLocker locker(&m_mutex); | |
| 107 |
2/2✓ Branch 3 → 4 taken 37 times.
✓ Branch 4 → 5 taken 37 times.
|
37 | m_evidence.correct(FolderEvidence::fromHeader(header), chosenFolder); |
| 108 | 37 | } | |
| 109 | |||
| 110 | ✗ | void FolderEvidenceModel::renameFolder(const QString &from, const QString &to) { | |
| 111 | ✗ | QMutexLocker locker(&m_mutex); | |
| 112 | ✗ | m_evidence.renameFolder(from, to); | |
| 113 | ✗ | } | |
| 114 | |||
| 115 | 616 | FolderSuggestion FolderEvidenceModel::suggest(const MailHeader &header, const QString &junkFolder) const { | |
| 116 | 616 | QMutexLocker locker(&m_mutex); | |
| 117 |
2/2✓ Branch 3 → 4 taken 616 times.
✓ Branch 4 → 5 taken 616 times.
|
616 | const QList<FolderEvidence::Candidate> ranked = m_evidence.rank(FolderEvidence::fromHeader(header)); |
| 118 | 616 | QList<FolderSuggestion::Candidate> scores; | |
| 119 |
5/5✓ Branch 6 → 7 taken 616 times.
✓ Branch 7 → 8 taken 616 times.
✓ Branch 8 → 9 taken 616 times.
✓ Branch 17 → 10 taken 433 times.
✓ Branch 17 → 18 taken 616 times.
|
1049 | for (const FolderEvidence::Candidate &candidate : ranked.mid(0, 2)) |
| 120 |
2/2✓ Branch 11 → 12 taken 433 times.
✓ Branch 13 → 14 taken 433 times.
|
1049 | scores.append({candidate.folder, m_calibration.confidence(candidate.kind, candidate.strength)}); |
| 121 |
1/1✓ Branch 19 → 20 taken 616 times.
|
616 | FolderSuggestion suggestion = FolderSuggestion::fromScores(scores, header.isSpam, junkFolder); |
| 122 | 616 | suggestion.threshold = m_stats.threshold; | |
| 123 |
5/6✓ Branch 21 → 22 taken 334 times.
✓ Branch 21 → 26 taken 282 times.
✓ Branch 24 → 25 taken 334 times.
✗ Branch 24 → 26 not taken.
✓ Branch 27 → 28 taken 334 times.
✓ Branch 27 → 35 taken 282 times.
|
616 | if (!ranked.isEmpty() && suggestion.primary.first == ranked.first().folder) { |
| 124 | 334 | suggestion.reason = int(ranked.first().kind); | |
| 125 | 334 | suggestion.support = ranked.first().support; | |
| 126 | 334 | suggestion.total = ranked.first().total; | |
| 127 | // The alternate is never shown as more certain than the primary. | ||
| 128 |
2/2✓ Branch 32 → 33 taken 99 times.
✓ Branch 32 → 42 taken 235 times.
|
334 | if (!suggestion.alternate.first.isEmpty()) |
| 129 | 99 | suggestion.alternate.second = std::min(suggestion.alternate.second, suggestion.primary.second); | |
| 130 |
4/6✓ Branch 35 → 36 taken 1 time.
✓ Branch 35 → 39 taken 281 times.
✗ Branch 37 → 38 not taken.
✓ Branch 37 → 39 taken 1 time.
✗ Branch 40 → 41 not taken.
✓ Branch 40 → 42 taken 282 times.
|
282 | } else if (header.isSpam && suggestion.primary.first == junkFolder) { |
| 131 | ✗ | suggestion.reason = FolderSuggestion::SpamFilter; | |
| 132 | } | ||
| 133 | 616 | return suggestion; | |
| 134 | 616 | } | |
| 135 | |||
| 136 | ✗ | FolderEvidenceModel::Stats FolderEvidenceModel::stats() const { | |
| 137 | ✗ | QMutexLocker locker(&m_mutex); | |
| 138 | ✗ | return m_stats; | |
| 139 | ✗ | } | |
| 140 |