data/FolderEvidence.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #include "FolderEvidence.h" | ||
| 2 | |||
| 3 | #include "MailAddressIndex.h" | ||
| 4 | #include "Models.h" | ||
| 5 | #include "util/MessageId.h" | ||
| 6 | |||
| 7 | #include <QRegularExpression> | ||
| 8 | #include <QSet> | ||
| 9 | #include <algorithm> | ||
| 10 | #include <cmath> | ||
| 11 | |||
| 12 | namespace { | ||
| 13 | constexpr double kAliasShare = 0.8; // an address decides only when it is this concentrated | ||
| 14 | constexpr int kAliasSupport = 3; | ||
| 15 | // An address on more than this share of all filings is the user's own main | ||
| 16 | // address (every folder gets its mail) — it never decides, also while the | ||
| 17 | // history is still too short to show its spread. | ||
| 18 | constexpr double kAliasMainShare = 0.5; | ||
| 19 | constexpr double kWordPrior = 0.1; // Laplace alpha of the subject words | ||
| 20 | |||
| 21 | 188886 | QStringList subjectWords(const QString &subject) { | |
| 22 | 28 | static const QRegularExpression prefix(QStringLiteral("^\\s*(re|fwd?|aw|wg|sv|tr)\\s*:\\s*"), | |
| 23 |
4/7✓ Branch 2 → 3 taken 14 times.
✓ Branch 2 → 15 taken 188872 times.
✓ Branch 4 → 5 taken 14 times.
✗ Branch 4 → 15 not taken.
✓ Branch 9 → 10 taken 14 times.
✗ Branch 69 → 70 not taken.
✗ Branch 69 → 71 not taken.
|
188914 | QRegularExpression::CaseInsensitiveOption); |
| 24 |
4/7✓ Branch 15 → 16 taken 14 times.
✓ Branch 15 → 28 taken 188872 times.
✓ Branch 17 → 18 taken 14 times.
✗ Branch 17 → 28 not taken.
✓ Branch 22 → 23 taken 14 times.
✗ Branch 79 → 80 not taken.
✗ Branch 79 → 81 not taken.
|
188900 | static const QRegularExpression split(QStringLiteral("[^\\p{L}\\p{N}]+")); |
| 25 | 188886 | QString text = subject; | |
| 26 |
4/4✓ Branch 31 → 32 taken 1406 times.
✓ Branch 34 → 35 taken 190292 times.
✓ Branch 35 → 30 taken 1406 times.
✓ Branch 35 → 36 taken 188886 times.
|
190292 | while (text.contains(prefix)) text.replace(prefix, QString()); |
| 27 | 188886 | QStringList words; | |
| 28 |
6/6✓ Branch 36 → 37 taken 188886 times.
✓ Branch 38 → 39 taken 188886 times.
✓ Branch 40 → 41 taken 188886 times.
✓ Branch 41 → 42 taken 188886 times.
✓ Branch 54 → 43 taken 387508 times.
✓ Branch 54 → 55 taken 188886 times.
|
576394 | for (const QString &word : text.toLower().split(split, Qt::SkipEmptyParts)) |
| 29 |
7/7✓ Branch 45 → 46 taken 385966 times.
✓ Branch 45 → 49 taken 1542 times.
✓ Branch 47 → 48 taken 359973 times.
✓ Branch 47 → 49 taken 25993 times.
✓ Branch 50 → 51 taken 359973 times.
✓ Branch 50 → 52 taken 27535 times.
✓ Branch 51 → 52 taken 359973 times.
|
576394 | if (word.size() >= 3 && !words.contains(word)) words << word; |
| 30 |
1/1✓ Branch 56 → 57 taken 188886 times.
|
377772 | return words.mid(0, 24); |
| 31 | 188886 | } | |
| 32 | |||
| 33 | // Folders of a count map as shares, largest first. | ||
| 34 | 233756 | QList<FolderEvidence::Candidate> shares(const QHash<QString, int> &counts, FolderEvidence::Kind kind) { | |
| 35 | 233756 | int total = 0; | |
| 36 |
2/2✓ Branch 7 → 4 taken 1313842 times.
✓ Branch 7 → 8 taken 233756 times.
|
1547598 | for (const int count : counts) total += count; |
| 37 | 233756 | QList<FolderEvidence::Candidate> result; | |
| 38 |
2/2✓ Branch 21 → 9 taken 1313842 times.
✓ Branch 21 → 22 taken 233756 times.
|
1547598 | for (auto it = counts.cbegin(); it != counts.cend(); ++it) |
| 39 |
1/2✓ Branch 10 → 11 taken 1313842 times.
✗ Branch 10 → 18 not taken.
|
2627684 | if (it.value() > 0) result.append({it.key(), kind, double(it.value()) / total, it.value(), total}); |
| 40 |
3/3✓ Branch 22 → 23 taken 233756 times.
✓ Branch 23 → 24 taken 233756 times.
✓ Branch 24 → 25 taken 233756 times.
|
233756 | std::sort(result.begin(), result.end(), [](const auto &a, const auto &b) { |
| 41 |
2/2✓ Branch 2 → 3 taken 1346741 times.
✓ Branch 2 → 4 taken 2881054 times.
|
4227795 | return a.strength != b.strength ? a.strength > b.strength : a.folder < b.folder; |
| 42 | }); | ||
| 43 | 233756 | return result; | |
| 44 | 1313842 | } | |
| 45 | } // namespace | ||
| 46 | |||
| 47 | 5561 | FolderEvidence::Mail FolderEvidence::fromHeader(const MailHeader &header) { | |
| 48 | 5561 | Mail mail; | |
| 49 |
1/1✓ Branch 2 → 3 taken 5561 times.
|
5561 | mail.messageId = MessageId::canonicalFirst(header.messageId); |
| 50 |
5/5✓ Branch 5 → 6 taken 5561 times.
✓ Branch 6 → 7 taken 5561 times.
✓ Branch 7 → 8 taken 5561 times.
✓ Branch 22 → 9 taken 789 times.
✓ Branch 22 → 23 taken 5561 times.
|
6350 | for (const QString &id : header.inReplyToIds + header.references) { |
| 51 |
1/1✓ Branch 10 → 11 taken 789 times.
|
789 | const QString canonical = MessageId::canonicalFirst(id); |
| 52 |
6/7✓ Branch 12 → 13 taken 789 times.
✗ Branch 12 → 16 not taken.
✓ Branch 14 → 15 taken 773 times.
✓ Branch 14 → 16 taken 16 times.
✓ Branch 17 → 18 taken 773 times.
✓ Branch 17 → 19 taken 16 times.
✓ Branch 18 → 19 taken 773 times.
|
789 | if (!canonical.isEmpty() && !mail.threadIds.contains(canonical)) mail.threadIds << canonical; |
| 53 | 6350 | } | |
| 54 |
2/2✓ Branch 25 → 26 taken 740 times.
✓ Branch 25 → 37 taken 4821 times.
|
5561 | if (!header.inReplyTo.isEmpty()) { |
| 55 |
1/1✓ Branch 26 → 27 taken 740 times.
|
740 | const QString canonical = MessageId::canonicalFirst(header.inReplyTo); |
| 56 |
3/7✓ Branch 28 → 29 taken 740 times.
✗ Branch 28 → 32 not taken.
✗ Branch 30 → 31 not taken.
✓ Branch 30 → 32 taken 740 times.
✗ Branch 33 → 34 not taken.
✓ Branch 33 → 35 taken 740 times.
✗ Branch 34 → 35 not taken.
|
740 | if (!canonical.isEmpty() && !mail.threadIds.contains(canonical)) mail.threadIds << canonical; |
| 57 | 740 | } | |
| 58 |
2/2✓ Branch 37 → 38 taken 5561 times.
✓ Branch 38 → 39 taken 5561 times.
|
5561 | mail.listId = header.listId.trimmed().toLower(); |
| 59 |
1/1✓ Branch 42 → 43 taken 5561 times.
|
5561 | const QStringList senders = MailAddressIndex::parse(header.from); |
| 60 |
1/1✓ Branch 43 → 44 taken 5561 times.
|
5561 | mail.sender = senders.value(0); |
| 61 |
3/3✓ Branch 46 → 47 taken 5561 times.
✓ Branch 47 → 48 taken 5561 times.
✓ Branch 48 → 49 taken 5561 times.
|
5561 | mail.recipients = MailAddressIndex::parse(header.to) + MailAddressIndex::parse(header.cc); |
| 62 | 5561 | mail.subject = header.subject; | |
| 63 |
4/4✓ Branch 54 → 55 taken 5561 times.
✓ Branch 55 → 56 taken 5540 times.
✓ Branch 55 → 58 taken 21 times.
✓ Branch 56 → 57 taken 5540 times.
|
5561 | mail.date = header.date.isValid() ? header.date.toSecsSinceEpoch() : 0; |
| 64 | 5561 | return mail; | |
| 65 | 5561 | } | |
| 66 | |||
| 67 | 104940 | void FolderEvidence::learn(const Mail &mail, const QString &folder) { | |
| 68 |
1/2✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 104940 times.
|
104940 | if (folder.isEmpty()) return; |
| 69 | 104940 | ++m_filings; | |
| 70 |
3/3✓ Branch 6 → 7 taken 104218 times.
✓ Branch 6 → 8 taken 722 times.
✓ Branch 7 → 8 taken 104218 times.
|
104940 | if (!mail.messageId.isEmpty()) m_threadFolder.insert(mail.messageId, folder); |
| 71 |
2/2✓ Branch 9 → 10 taken 20875 times.
✓ Branch 9 → 13 taken 84065 times.
|
104940 | if (!mail.listId.isEmpty()) { |
| 72 |
2/2✓ Branch 10 → 11 taken 20875 times.
✓ Branch 11 → 12 taken 20875 times.
|
20875 | ++m_listFolders[mail.listId][folder]; |
| 73 |
1/1✓ Branch 12 → 13 taken 20875 times.
|
20875 | m_listOverride.remove(mail.listId); |
| 74 | } | ||
| 75 |
1/2✓ Branch 14 → 15 taken 104940 times.
✗ Branch 14 → 23 not taken.
|
104940 | if (!mail.sender.isEmpty()) { |
| 76 |
1/1✓ Branch 15 → 16 taken 104940 times.
|
104940 | QList<Placement> &history = m_senderHistory[mail.sender]; |
| 77 |
1/1✓ Branch 17 → 18 taken 104940 times.
|
104940 | history.append({mail.date, folder}); |
| 78 |
2/2✓ Branch 20 → 21 taken 156 times.
✓ Branch 20 → 22 taken 104784 times.
|
104940 | if (history.size() > m_options.maxSenderHistory) history.removeFirst(); |
| 79 |
1/1✓ Branch 22 → 23 taken 104940 times.
|
104940 | m_senderOverride.remove(mail.sender); |
| 80 | } | ||
| 81 |
4/4✓ Branch 26 → 27 taken 104934 times.
✓ Branch 27 → 28 taken 104934 times.
✓ Branch 30 → 25 taken 104934 times.
✓ Branch 30 → 31 taken 104940 times.
|
209874 | for (const QString &recipient : mail.recipients) ++m_aliasFolders[recipient][folder]; |
| 82 |
1/1✓ Branch 31 → 32 taken 104940 times.
|
104940 | const QStringList words = subjectWords(mail.subject); |
| 83 |
4/4✓ Branch 35 → 36 taken 200458 times.
✓ Branch 36 → 37 taken 200458 times.
✓ Branch 39 → 34 taken 200458 times.
✓ Branch 39 → 40 taken 104940 times.
|
305398 | for (const QString &word : words) ++m_wordFolders[word][folder]; |
| 84 |
1/1✓ Branch 40 → 41 taken 104940 times.
|
104940 | ++m_folderDocs[folder]; |
| 85 |
1/1✓ Branch 42 → 43 taken 104940 times.
|
104940 | m_folderWords[folder] += int(words.size()); |
| 86 | 209880 | } | |
| 87 | |||
| 88 | 38 | void FolderEvidence::correct(const Mail &mail, const QString &chosenFolder) { | |
| 89 |
1/2✓ Branch 3 → 4 taken 38 times.
✗ Branch 3 → 5 not taken.
|
38 | if (!mail.sender.isEmpty()) m_senderOverride.insert(mail.sender, chosenFolder); |
| 90 |
1/2✗ Branch 6 → 7 not taken.
✓ Branch 6 → 8 taken 38 times.
|
38 | if (!mail.listId.isEmpty()) m_listOverride.insert(mail.listId, chosenFolder); |
| 91 | 38 | learn(mail, chosenFolder); | |
| 92 | // learn() cleared the overrides of a regular filing; a correction keeps them. | ||
| 93 |
1/2✓ Branch 10 → 11 taken 38 times.
✗ Branch 10 → 12 not taken.
|
38 | if (!mail.sender.isEmpty()) m_senderOverride.insert(mail.sender, chosenFolder); |
| 94 |
1/2✗ Branch 13 → 14 not taken.
✓ Branch 13 → 15 taken 38 times.
|
38 | if (!mail.listId.isEmpty()) m_listOverride.insert(mail.listId, chosenFolder); |
| 95 | 38 | } | |
| 96 | |||
| 97 | 1 | void FolderEvidence::renameFolder(const QString &from, const QString &to) { | |
| 98 | 1 | const auto renameIn = [&](QHash<QString, int> &counts) { | |
| 99 |
1/2✗ Branch 3 → 4 not taken.
✓ Branch 3 → 5 taken 1 time.
|
1 | if (!counts.contains(from)) return; |
| 100 | 1 | counts[to] += counts.take(from); | |
| 101 | 1 | }; | |
| 102 |
2/5✓ Branch 2 → 3 taken 1 time.
✗ Branch 6 → 7 not taken.
✗ Branch 6 → 8 not taken.
✗ Branch 10 → 4 not taken.
✓ Branch 10 → 11 taken 1 time.
|
1 | for (QString &folder : m_threadFolder) if (folder == from) folder = to; |
| 103 |
4/4✓ Branch 11 → 12 taken 1 time.
✓ Branch 14 → 15 taken 1 time.
✓ Branch 17 → 13 taken 1 time.
✓ Branch 17 → 18 taken 1 time.
|
2 | for (auto &counts : m_listFolders) renameIn(counts); |
| 104 |
2/4✓ Branch 18 → 19 taken 1 time.
✗ Branch 21 → 22 not taken.
✗ Branch 24 → 20 not taken.
✓ Branch 24 → 25 taken 1 time.
|
1 | for (auto &counts : m_aliasFolders) renameIn(counts); |
| 105 |
2/4✓ Branch 25 → 26 taken 1 time.
✗ Branch 28 → 29 not taken.
✗ Branch 31 → 27 not taken.
✓ Branch 31 → 32 taken 1 time.
|
1 | for (auto &counts : m_wordFolders) renameIn(counts); |
| 106 |
3/3✓ Branch 32 → 33 taken 1 time.
✓ Branch 47 → 34 taken 1 time.
✓ Branch 47 → 48 taken 1 time.
|
2 | for (auto &history : m_senderHistory) |
| 107 |
5/6✓ Branch 35 → 36 taken 1 time.
✓ Branch 36 → 37 taken 1 time.
✓ Branch 40 → 41 taken 1 time.
✗ Branch 40 → 42 not taken.
✓ Branch 44 → 38 taken 1 time.
✓ Branch 44 → 45 taken 1 time.
|
2 | for (Placement &placement : history) if (placement.folder == from) placement.folder = to; |
| 108 |
2/5✓ Branch 48 → 49 taken 1 time.
✗ Branch 52 → 53 not taken.
✗ Branch 52 → 54 not taken.
✗ Branch 56 → 50 not taken.
✓ Branch 56 → 57 taken 1 time.
|
1 | for (QString &folder : m_senderOverride) if (folder == from) folder = to; |
| 109 |
2/5✓ Branch 57 → 58 taken 1 time.
✗ Branch 61 → 62 not taken.
✗ Branch 61 → 63 not taken.
✗ Branch 65 → 59 not taken.
✓ Branch 65 → 66 taken 1 time.
|
1 | for (QString &folder : m_listOverride) if (folder == from) folder = to; |
| 110 |
3/4✓ Branch 67 → 68 taken 1 time.
✗ Branch 67 → 71 not taken.
✓ Branch 68 → 69 taken 1 time.
✓ Branch 69 → 70 taken 1 time.
|
1 | if (m_folderDocs.contains(from)) m_folderDocs[to] += m_folderDocs.take(from); |
| 111 |
3/4✓ Branch 72 → 73 taken 1 time.
✗ Branch 72 → 76 not taken.
✓ Branch 73 → 74 taken 1 time.
✓ Branch 74 → 75 taken 1 time.
|
1 | if (m_folderWords.contains(from)) m_folderWords[to] += m_folderWords.take(from); |
| 112 | 1 | } | |
| 113 | |||
| 114 | 1 | void FolderEvidence::removeFolder(const QString &folder) { | |
| 115 | 1 | m_threadFolder.removeIf([&](const auto &it) { return it.value() == folder; }); | |
| 116 |
4/4✓ Branch 3 → 4 taken 1 time.
✓ Branch 6 → 7 taken 1 time.
✓ Branch 9 → 5 taken 1 time.
✓ Branch 9 → 10 taken 1 time.
|
2 | for (auto &counts : m_listFolders) counts.remove(folder); |
| 117 |
2/4✓ Branch 10 → 11 taken 1 time.
✗ Branch 13 → 14 not taken.
✗ Branch 16 → 12 not taken.
✓ Branch 16 → 17 taken 1 time.
|
1 | for (auto &counts : m_aliasFolders) counts.remove(folder); |
| 118 |
2/4✓ Branch 17 → 18 taken 1 time.
✗ Branch 20 → 21 not taken.
✗ Branch 23 → 19 not taken.
✓ Branch 23 → 24 taken 1 time.
|
1 | for (auto &counts : m_wordFolders) counts.remove(folder); |
| 119 |
4/4✓ Branch 24 → 25 taken 1 time.
✓ Branch 27 → 28 taken 1 time.
✓ Branch 30 → 26 taken 1 time.
✓ Branch 30 → 31 taken 1 time.
|
3 | for (auto &history : m_senderHistory) history.removeIf([&](const Placement &p) { return p.folder == folder; }); |
| 120 | 1 | m_senderOverride.removeIf([&](const auto &it) { return it.value() == folder; }); | |
| 121 | 1 | m_listOverride.removeIf([&](const auto &it) { return it.value() == folder; }); | |
| 122 | 1 | m_folderDocs.remove(folder); | |
| 123 | 1 | m_folderWords.remove(folder); | |
| 124 | 1 | } | |
| 125 | |||
| 126 | 83946 | QList<FolderEvidence::Candidate> FolderEvidence::subjectCandidates(const Mail &mail) const { | |
| 127 |
1/1✓ Branch 2 → 3 taken 83946 times.
|
83946 | const QStringList words = subjectWords(mail.subject); |
| 128 |
6/6✓ Branch 4 → 5 taken 83943 times.
✓ Branch 4 → 7 taken 3 times.
✓ Branch 6 → 7 taken 317 times.
✓ Branch 6 → 8 taken 83626 times.
✓ Branch 9 → 10 taken 320 times.
✓ Branch 9 → 11 taken 83626 times.
|
83946 | if (words.isEmpty() || m_folderDocs.isEmpty()) return {}; |
| 129 | 83626 | const double vocabulary = std::max<double>(1, m_wordFolders.size()); | |
| 130 | 83626 | QList<QPair<QString, double>> scores; | |
| 131 | 83626 | double best = -1e300; | |
| 132 | 83626 | bool anyKnown = false; | |
| 133 |
2/2✓ Branch 39 → 14 taken 991735 times.
✓ Branch 39 → 40 taken 83626 times.
|
1075361 | for (auto it = m_folderDocs.cbegin(); it != m_folderDocs.cend(); ++it) { |
| 134 | 991735 | double score = std::log(double(it.value()) / m_filings); | |
| 135 | 991735 | const double denominator = m_folderWords.value(it.key()) + kWordPrior * vocabulary; | |
| 136 |
2/2✓ Branch 30 → 19 taken 1866388 times.
✓ Branch 30 → 31 taken 991735 times.
|
2858123 | for (const QString &word : words) { |
| 137 | 1866388 | const int count = m_wordFolders.value(word).value(it.key()); | |
| 138 |
4/4✓ Branch 24 → 25 taken 86905 times.
✓ Branch 24 → 26 taken 1779483 times.
✓ Branch 25 → 26 taken 83519 times.
✓ Branch 25 → 27 taken 3386 times.
|
1866388 | anyKnown = anyKnown || count > 0; |
| 139 | 1866388 | score += std::log((count + kWordPrior) / denominator); | |
| 140 | } | ||
| 141 |
1/1✓ Branch 33 → 34 taken 991735 times.
|
991735 | scores.append({it.key(), score}); |
| 142 | 991735 | best = std::max(best, score); | |
| 143 | } | ||
| 144 |
2/2✓ Branch 40 → 41 taken 107 times.
✓ Branch 40 → 42 taken 83519 times.
|
83626 | if (!anyKnown) return {}; |
| 145 | 83519 | double sum = 0; | |
| 146 |
4/4✓ Branch 42 → 43 taken 83519 times.
✓ Branch 43 → 44 taken 83519 times.
✓ Branch 48 → 45 taken 991420 times.
✓ Branch 48 → 49 taken 83519 times.
|
1074939 | for (auto &score : scores) sum += (score.second = std::exp(score.second - best)); |
| 147 | 83519 | QList<Candidate> result; | |
| 148 |
4/4✓ Branch 49 → 50 taken 83519 times.
✓ Branch 50 → 51 taken 83519 times.
✓ Branch 58 → 52 taken 991420 times.
✓ Branch 58 → 59 taken 83519 times.
|
1074939 | for (const auto &score : scores) result.append({score.first, Kind::Subject, score.second / sum, 0, m_filings}); |
| 149 |
3/3✓ Branch 59 → 60 taken 83519 times.
✓ Branch 60 → 61 taken 83519 times.
✓ Branch 61 → 62 taken 83519 times.
|
83519 | std::sort(result.begin(), result.end(), [](const auto &a, const auto &b) { |
| 150 |
2/2✓ Branch 2 → 3 taken 3396580 times.
✓ Branch 2 → 4 taken 320739 times.
|
3717319 | return a.strength != b.strength ? a.strength > b.strength : a.folder < b.folder; |
| 151 | }); | ||
| 152 |
1/1✓ Branch 62 → 63 taken 83519 times.
|
83519 | return result.mid(0, 3); |
| 153 | 1075366 | } | |
| 154 | |||
| 155 | 106510 | QList<FolderEvidence::Candidate> FolderEvidence::rank(const Mail &mail) const { | |
| 156 | 106510 | QList<Candidate> all; | |
| 157 | // 1. Thread. | ||
| 158 | 106510 | QHash<QString, int> thread; | |
| 159 |
2/2✓ Branch 14 → 4 taken 773 times.
✓ Branch 14 → 15 taken 106510 times.
|
107283 | for (const QString &id : mail.threadIds) |
| 160 |
3/3✓ Branch 8 → 9 taken 745 times.
✓ Branch 8 → 12 taken 28 times.
✓ Branch 10 → 11 taken 745 times.
|
773 | if (const auto it = m_threadFolder.constFind(id); it != m_threadFolder.constEnd()) ++thread[it.value()]; |
| 161 |
2/2✓ Branch 15 → 16 taken 106510 times.
✓ Branch 16 → 17 taken 106510 times.
|
106510 | all += shares(thread, Kind::Thread); |
| 162 | // 2. Mailing list. | ||
| 163 |
2/2✓ Branch 19 → 20 taken 21076 times.
✓ Branch 19 → 39 taken 85434 times.
|
106510 | if (!mail.listId.isEmpty()) { |
| 164 |
1/2✗ Branch 23 → 24 not taken.
✓ Branch 23 → 29 taken 21076 times.
|
21076 | if (const auto it = m_listOverride.constFind(mail.listId); it != m_listOverride.constEnd()) |
| 165 | ✗ | all.append({it.value(), Kind::List, 1.0, 1, 1}); | |
| 166 |
2/2✓ Branch 32 → 33 taken 21065 times.
✓ Branch 32 → 38 taken 11 times.
|
21076 | if (const auto it = m_listFolders.constFind(mail.listId); it != m_listFolders.constEnd()) |
| 167 |
2/2✓ Branch 34 → 35 taken 21065 times.
✓ Branch 35 → 36 taken 21065 times.
|
21065 | all += shares(it.value(), Kind::List); |
| 168 | } | ||
| 169 | // 3. Sender, recency-weighted. | ||
| 170 |
2/2✓ Branch 40 → 41 taken 106506 times.
✓ Branch 40 → 100 taken 4 times.
|
106510 | if (!mail.sender.isEmpty()) { |
| 171 |
2/2✓ Branch 44 → 45 taken 22 times.
✓ Branch 44 → 50 taken 106484 times.
|
106506 | if (const auto it = m_senderOverride.constFind(mail.sender); it != m_senderOverride.constEnd()) |
| 172 | 22 | all.append({it.value(), Kind::Sender, 1.0, 1, 1}); | |
| 173 |
2/2✓ Branch 53 → 54 taken 4741 times.
✓ Branch 53 → 99 taken 101765 times.
|
106506 | if (const auto it = m_senderHistory.constFind(mail.sender); it != m_senderHistory.constEnd()) { |
| 174 |
4/4✓ Branch 54 → 55 taken 4736 times.
✓ Branch 54 → 56 taken 5 times.
✓ Branch 58 → 59 taken 1 time.
✓ Branch 58 → 60 taken 4 times.
|
4741 | const qint64 now = mail.date > 0 ? mail.date : (it->isEmpty() ? 0 : it->last().date); |
| 175 | 4741 | QHash<QString, double> weights; | |
| 176 | 4741 | QHash<QString, int> counts; | |
| 177 | 4741 | double total = 0; | |
| 178 |
2/2✓ Branch 73 → 67 taken 38779 times.
✓ Branch 73 → 74 taken 4741 times.
|
43520 | for (const Placement &placement : *it) { |
| 179 | 38779 | const double ageDays = std::max<double>(0, double(now - placement.date) / 86400.0); | |
| 180 | 38779 | const double weight = std::pow(0.5, ageDays / m_options.senderHalfLifeDays); | |
| 181 |
1/1✓ Branch 69 → 70 taken 38779 times.
|
38779 | weights[placement.folder] += weight; |
| 182 |
1/1✓ Branch 70 → 71 taken 38779 times.
|
38779 | ++counts[placement.folder]; |
| 183 | 38779 | total += weight; | |
| 184 | } | ||
| 185 | 4741 | QList<Candidate> sender; | |
| 186 |
2/2✓ Branch 90 → 75 taken 6861 times.
✓ Branch 90 → 91 taken 4741 times.
|
11602 | for (auto w = weights.cbegin(); w != weights.cend(); ++w) |
| 187 | 27444 | sender.append({w.key(), Kind::Sender, total > 0 ? w.value() / total : 0, counts.value(w.key()), int(it->size())}); | |
| 188 |
3/3✓ Branch 91 → 92 taken 4741 times.
✓ Branch 92 → 93 taken 4741 times.
✓ Branch 93 → 94 taken 4741 times.
|
4741 | std::sort(sender.begin(), sender.end(), [](const auto &a, const auto &b) { |
| 189 |
2/2✓ Branch 2 → 3 taken 2858 times.
✓ Branch 2 → 4 taken 242 times.
|
3100 | return a.strength != b.strength ? a.strength > b.strength : a.folder < b.folder; |
| 190 | }); | ||
| 191 |
1/1✓ Branch 94 → 95 taken 4741 times.
|
4741 | all += sender; |
| 192 | 4741 | } | |
| 193 | } | ||
| 194 | // 4. Recipient alias: only a concentrated address decides. | ||
| 195 | 106510 | Candidate alias; | |
| 196 |
2/2✓ Branch 139 → 102 taken 106514 times.
✓ Branch 139 → 140 taken 106510 times.
|
213024 | for (const QString &recipient : mail.recipients) { |
| 197 | 106514 | const auto it = m_aliasFolders.constFind(recipient); | |
| 198 |
2/2✓ Branch 106 → 107 taken 333 times.
✓ Branch 106 → 108 taken 106181 times.
|
212426 | if (it == m_aliasFolders.constEnd()) continue; |
| 199 |
1/1✓ Branch 109 → 110 taken 106181 times.
|
106181 | const QList<Candidate> ranked = shares(it.value(), Kind::Alias); |
| 200 |
5/6✓ Branch 111 → 112 taken 106181 times.
✗ Branch 111 → 115 not taken.
✓ Branch 113 → 114 taken 105912 times.
✓ Branch 113 → 115 taken 269 times.
✓ Branch 116 → 117 taken 105912 times.
✓ Branch 116 → 118 taken 269 times.
|
106181 | if (!ranked.isEmpty() && ranked.first().total > kAliasMainShare * m_filings) continue; |
| 201 |
6/8✓ Branch 119 → 120 taken 269 times.
✗ Branch 119 → 127 not taken.
✓ Branch 121 → 122 taken 261 times.
✓ Branch 121 → 127 taken 8 times.
✓ Branch 123 → 124 taken 261 times.
✗ Branch 123 → 127 not taken.
✓ Branch 128 → 129 taken 261 times.
✓ Branch 128 → 131 taken 8 times.
|
530 | if (!ranked.isEmpty() && ranked.first().total >= kAliasSupport && ranked.first().strength >= kAliasShare && |
| 202 |
1/2✓ Branch 125 → 126 taken 261 times.
✗ Branch 125 → 127 not taken.
|
261 | ranked.first().strength > alias.strength) |
| 203 | 261 | alias = ranked.first(); | |
| 204 | 106181 | } | |
| 205 |
3/3✓ Branch 140 → 141 taken 261 times.
✓ Branch 140 → 142 taken 106249 times.
✓ Branch 141 → 142 taken 261 times.
|
106510 | if (alias.kind == Kind::Alias) all.append(alias); |
| 206 | // 5. Subject words: only when the stronger evidence leaves no alternative. | ||
| 207 | 106510 | QSet<QString> folders; | |
| 208 |
3/3✓ Branch 147 → 148 taken 69313 times.
✓ Branch 150 → 146 taken 69313 times.
✓ Branch 150 → 151 taken 106510 times.
|
175823 | for (const Candidate &candidate : std::as_const(all)) folders.insert(candidate.folder); |
| 209 |
4/4✓ Branch 152 → 153 taken 83946 times.
✓ Branch 152 → 157 taken 22564 times.
✓ Branch 153 → 154 taken 83946 times.
✓ Branch 154 → 155 taken 83946 times.
|
106510 | if (folders.size() < 2) all += subjectCandidates(mail); |
| 210 | // Distinct folders, the first (strongest) evidence of each. | ||
| 211 | 106510 | QList<Candidate> result; | |
| 212 | 106510 | QSet<QString> seen; | |
| 213 |
2/2✓ Branch 174 → 161 taken 318798 times.
✓ Branch 174 → 175 taken 106510 times.
|
425308 | for (const Candidate &candidate : std::as_const(all)) { |
| 214 |
5/6✓ Branch 163 → 164 taken 318798 times.
✗ Branch 163 → 166 not taken.
✓ Branch 165 → 166 taken 3436 times.
✓ Branch 165 → 167 taken 315362 times.
✓ Branch 168 → 169 taken 3436 times.
✓ Branch 168 → 170 taken 315362 times.
|
318798 | if (candidate.folder.isEmpty() || seen.contains(candidate.folder)) continue; |
| 215 |
1/1✓ Branch 170 → 171 taken 315362 times.
|
315362 | seen.insert(candidate.folder); |
| 216 |
1/1✓ Branch 171 → 172 taken 315362 times.
|
315362 | result.append(candidate); |
| 217 | } | ||
| 218 | 106510 | return result; | |
| 219 | 133976 | } | |
| 220 | |||
| 221 |
1/1✓ Branch 2 → 3 taken 209988 times.
|
209988 | int FolderCalibration::bin(double strength) { return std::clamp(int(strength * 10), 0, 9); } |
| 222 | |||
| 223 | 104786 | void FolderCalibration::record(FolderEvidence::Kind kind, double strength, bool correct) { | |
| 224 |
2/2✓ Branch 2 → 3 taken 104786 times.
✓ Branch 3 → 4 taken 104786 times.
|
104786 | auto &entry = m_bins[int(kind) * 16 + bin(strength)]; |
| 225 |
2/2✓ Branch 4 → 5 taken 8681 times.
✓ Branch 4 → 6 taken 96105 times.
|
104786 | entry.first += correct ? 1 : 0; |
| 226 | 104786 | ++entry.second; | |
| 227 | 104786 | } | |
| 228 | |||
| 229 | 105202 | double FolderCalibration::confidence(FolderEvidence::Kind kind, double strength) const { | |
| 230 | // The bin, else the kind as a whole, else a prior per kind; Laplace (1, 2). | ||
| 231 |
1/1✓ Branch 2 → 3 taken 105202 times.
|
105202 | const auto it = m_bins.constFind(int(kind) * 16 + bin(strength)); |
| 232 |
6/6✓ Branch 6 → 7 taken 105011 times.
✓ Branch 6 → 10 taken 191 times.
✓ Branch 8 → 9 taken 104632 times.
✓ Branch 8 → 10 taken 379 times.
✓ Branch 11 → 12 taken 104632 times.
✓ Branch 11 → 15 taken 570 times.
|
105202 | if (it != m_bins.constEnd() && it->second >= 5) return (it->first + 1.0) / (it->second + 2.0); |
| 233 | 570 | int correct = 0, total = 0; | |
| 234 |
2/2✓ Branch 18 → 16 taken 5700 times.
✓ Branch 18 → 19 taken 570 times.
|
6270 | for (int b = 0; b < 10; ++b) { |
| 235 | 5700 | const auto entry = m_bins.value(int(kind) * 16 + b); | |
| 236 | 5700 | correct += entry.first; | |
| 237 | 5700 | total += entry.second; | |
| 238 | } | ||
| 239 |
2/2✓ Branch 19 → 20 taken 295 times.
✓ Branch 19 → 21 taken 275 times.
|
570 | if (total >= 5) return (correct + 1.0) / (total + 2.0); |
| 240 |
5/7✓ Branch 21 → 22 taken 9 times.
✓ Branch 21 → 23 taken 10 times.
✓ Branch 21 → 24 taken 191 times.
✓ Branch 21 → 25 taken 5 times.
✓ Branch 21 → 26 taken 60 times.
✗ Branch 21 → 27 not taken.
✗ Branch 21 → 28 not taken.
|
275 | switch (kind) { |
| 241 | 9 | case FolderEvidence::Kind::Thread: return 0.9; | |
| 242 | 10 | case FolderEvidence::Kind::List: return 0.85; | |
| 243 | 191 | case FolderEvidence::Kind::Sender: return 0.75 * strength + 0.1; | |
| 244 | 5 | case FolderEvidence::Kind::Alias: return 0.7; | |
| 245 | 60 | case FolderEvidence::Kind::Subject: return 0.4 * strength; | |
| 246 | ✗ | case FolderEvidence::Kind::None: break; | |
| 247 | } | ||
| 248 | ✗ | return 0; | |
| 249 | } | ||
| 250 | |||
| 251 | 1 | QString FolderCalibration::serialize() const { | |
| 252 | 1 | QStringList parts; | |
| 253 |
1/1✓ Branch 2 → 3 taken 1 time.
|
1 | QList<int> keys = m_bins.keys(); |
| 254 |
3/3✓ Branch 3 → 4 taken 1 time.
✓ Branch 4 → 5 taken 1 time.
✓ Branch 5 → 6 taken 1 time.
|
1 | std::sort(keys.begin(), keys.end()); |
| 255 |
4/4✓ Branch 6 → 7 taken 1 time.
✓ Branch 7 → 8 taken 1 time.
✓ Branch 29 → 9 taken 1 time.
✓ Branch 29 → 30 taken 1 time.
|
2 | for (const int key : keys) |
| 256 |
4/4✓ Branch 14 → 15 taken 1 time.
✓ Branch 17 → 18 taken 1 time.
✓ Branch 20 → 21 taken 1 time.
✓ Branch 21 → 22 taken 1 time.
|
4 | parts << QStringLiteral("%1:%2:%3").arg(key).arg(m_bins.value(key).first).arg(m_bins.value(key).second); |
| 257 |
1/1✓ Branch 32 → 33 taken 1 time.
|
2 | return parts.join(QLatin1Char(',')); |
| 258 | 1 | } | |
| 259 | |||
| 260 | 2 | FolderCalibration FolderCalibration::deserialize(const QString &text) { | |
| 261 | 2 | FolderCalibration calibration; | |
| 262 |
5/5✓ Branch 5 → 6 taken 2 times.
✓ Branch 6 → 7 taken 2 times.
✓ Branch 7 → 8 taken 2 times.
✓ Branch 41 → 9 taken 4 times.
✓ Branch 41 → 42 taken 2 times.
|
6 | for (const QString &part : text.split(QLatin1Char(','), Qt::SkipEmptyParts)) { |
| 263 |
1/1✓ Branch 13 → 14 taken 4 times.
|
4 | const QStringList fields = part.split(QLatin1Char(':')); |
| 264 | 4 | bool keyOk = false, correctOk = false, totalOk = false; | |
| 265 |
2/2✓ Branch 15 → 16 taken 1 time.
✓ Branch 15 → 17 taken 3 times.
|
4 | if (fields.size() != 3) continue; |
| 266 |
3/3✓ Branch 18 → 19 taken 3 times.
✓ Branch 20 → 21 taken 3 times.
✓ Branch 22 → 23 taken 3 times.
|
3 | const int key = fields[0].toInt(&keyOk), correct = fields[1].toInt(&correctOk), total = fields[2].toInt(&totalOk); |
| 267 |
9/14✓ Branch 23 → 24 taken 3 times.
✗ Branch 23 → 33 not taken.
✓ Branch 24 → 25 taken 3 times.
✗ Branch 24 → 33 not taken.
✓ Branch 25 → 26 taken 3 times.
✗ Branch 25 → 33 not taken.
✓ Branch 26 → 27 taken 3 times.
✗ Branch 26 → 33 not taken.
✓ Branch 27 → 28 taken 2 times.
✓ Branch 27 → 33 taken 1 time.
✓ Branch 28 → 29 taken 1 time.
✓ Branch 28 → 33 taken 1 time.
✓ Branch 29 → 30 taken 1 time.
✗ Branch 29 → 33 not taken.
|
3 | if (keyOk && correctOk && totalOk && key >= 0 && key < 16 * 6 && total >= correct && correct >= 0) |
| 268 |
1/1✓ Branch 31 → 32 taken 1 time.
|
1 | calibration.m_bins.insert(key, {correct, total}); |
| 269 | 6 | } | |
| 270 | 2 | return calibration; | |
| 271 | ✗ | } | |
| 272 | |||
| 273 | 271 | QString FolderReplayMetrics::report() const { | |
| 274 | 271 | QStringList bins; | |
| 275 |
2/2✓ Branch 28 → 3 taken 2680 times.
✓ Branch 28 → 29 taken 271 times.
|
2951 | for (int b = 0; b < reliability.size(); ++b) |
| 276 |
2/2✓ Branch 4 → 5 taken 154 times.
✓ Branch 4 → 26 taken 2526 times.
|
2680 | if (reliability[b].second > 0) |
| 277 |
5/5✓ Branch 9 → 10 taken 154 times.
✓ Branch 11 → 12 taken 154 times.
✓ Branch 14 → 15 taken 154 times.
✓ Branch 17 → 18 taken 154 times.
✓ Branch 18 → 19 taken 154 times.
|
770 | bins << QStringLiteral("%1-%2%:%3/%4").arg(b * 10).arg(b * 10 + 10).arg(reliability[b].first).arg(reliability[b].second); |
| 278 | 542 | return QStringLiteral("mails %1 · coverage %2 % · precision of shown %3 % · top-2 %4 % · answered %5 % · reliability %6") | |
| 279 |
4/4✓ Branch 33 → 34 taken 271 times.
✓ Branch 36 → 37 taken 271 times.
✓ Branch 39 → 40 taken 271 times.
✓ Branch 42 → 43 taken 271 times.
|
1355 | .arg(mails).arg(coverage() * 100, 0, 'f', 1).arg(precision() * 100, 0, 'f', 1).arg(top2() * 100, 0, 'f', 1) |
| 280 |
5/5✓ Branch 44 → 45 taken 67 times.
✓ Branch 44 → 46 taken 204 times.
✓ Branch 47 → 48 taken 271 times.
✓ Branch 51 → 52 taken 271 times.
✓ Branch 52 → 53 taken 271 times.
|
813 | .arg(mails ? 100.0 * answered / mails : 0, 0, 'f', 1).arg(bins.join(QLatin1Char(' '))); |
| 281 | 271 | } | |
| 282 | |||
| 283 | 2 | double FolderReplayMetrics::thresholdFor(double precision) const { | |
| 284 | 2 | int correct = 0, total = 0; | |
| 285 | 2 | double threshold = 1.0; | |
| 286 |
2/2✓ Branch 13 → 4 taken 20 times.
✓ Branch 13 → 14 taken 2 times.
|
22 | for (int b = int(reliability.size()) - 1; b >= 0; --b) { |
| 287 |
2/2✓ Branch 5 → 6 taken 9 times.
✓ Branch 5 → 7 taken 11 times.
|
20 | if (reliability[b].second == 0) continue; |
| 288 | 11 | correct += reliability[b].first; | |
| 289 | 11 | total += reliability[b].second; | |
| 290 |
1/2✗ Branch 9 → 10 not taken.
✓ Branch 9 → 11 taken 11 times.
|
11 | if (double(correct) / total < precision) break; |
| 291 | 11 | threshold = b / 10.0; | |
| 292 | } | ||
| 293 | 2 | return threshold; | |
| 294 | } | ||
| 295 | |||
| 296 | 265 | QPair<double, double> FolderReplayMetrics::at(double threshold) const { | |
| 297 | 265 | int correct = 0, total = 0; | |
| 298 |
2/2✓ Branch 9 → 3 taken 2650 times.
✓ Branch 9 → 10 taken 265 times.
|
2915 | for (int b = 0; b < reliability.size(); ++b) |
| 299 |
2/2✓ Branch 3 → 4 taken 1589 times.
✓ Branch 3 → 7 taken 1061 times.
|
2650 | if (b / 10.0 >= threshold - 1e-9) { |
| 300 | 1589 | correct += reliability[b].first; | |
| 301 | 1589 | total += reliability[b].second; | |
| 302 | } | ||
| 303 |
4/4✓ Branch 10 → 11 taken 61 times.
✓ Branch 10 → 12 taken 204 times.
✓ Branch 13 → 14 taken 44 times.
✓ Branch 13 → 15 taken 221 times.
|
265 | return {mails ? double(total) / mails : 0, total ? double(correct) / total : 0}; |
| 304 | } | ||
| 305 | |||
| 306 | 266 | FolderReplayMetrics replayFolderEvidence(const QList<FolderReplayItem> &items, double threshold, | |
| 307 | FolderCalibration *calibration, FolderEvidence *trained) { | ||
| 308 | 266 | FolderEvidence local; | |
| 309 |
2/2✓ Branch 2 → 3 taken 1 time.
✓ Branch 2 → 4 taken 265 times.
|
266 | FolderEvidence &evidence = trained ? *trained : local; |
| 310 | 266 | FolderCalibration localCalibration; | |
| 311 |
1/2✗ Branch 5 → 6 not taken.
✓ Branch 5 → 7 taken 266 times.
|
266 | FolderCalibration &bins = calibration ? *calibration : localCalibration; |
| 312 | 266 | FolderReplayMetrics metrics; | |
| 313 |
1/1✓ Branch 9 → 10 taken 266 times.
|
266 | metrics.reliability = QList<QPair<int, int>>(10, {0, 0}); |
| 314 |
2/2✓ Branch 47 → 14 taken 104889 times.
✓ Branch 47 → 48 taken 266 times.
|
105155 | for (const FolderReplayItem &item : items) { |
| 315 | 104889 | ++metrics.mails; | |
| 316 |
1/1✓ Branch 15 → 16 taken 104889 times.
|
104889 | const QList<FolderEvidence::Candidate> candidates = evidence.rank(item.mail); |
| 317 |
2/2✓ Branch 17 → 18 taken 104766 times.
✓ Branch 17 → 43 taken 123 times.
|
104889 | if (!candidates.isEmpty()) { |
| 318 | 104766 | ++metrics.answered; | |
| 319 | 104766 | const FolderEvidence::Candidate &first = candidates.first(); | |
| 320 | 104766 | const bool correct = first.folder == item.folder; | |
| 321 |
1/1✓ Branch 20 → 21 taken 104766 times.
|
104766 | const double confidence = bins.confidence(first.kind, first.strength); |
| 322 |
1/1✓ Branch 21 → 22 taken 104766 times.
|
104766 | const int reliabilityBin = std::clamp(int(confidence * 10), 0, 9); |
| 323 |
3/3✓ Branch 22 → 23 taken 104766 times.
✓ Branch 23 → 24 taken 8663 times.
✓ Branch 23 → 25 taken 96103 times.
|
104766 | metrics.reliability[reliabilityBin].first += correct ? 1 : 0; |
| 324 |
1/1✓ Branch 26 → 27 taken 104766 times.
|
104766 | ++metrics.reliability[reliabilityBin].second; |
| 325 |
1/2✓ Branch 27 → 28 taken 104766 times.
✗ Branch 27 → 32 not taken.
|
104766 | if (confidence >= threshold) { |
| 326 | 104766 | ++metrics.shown; | |
| 327 |
2/2✓ Branch 28 → 29 taken 8663 times.
✓ Branch 28 → 30 taken 96103 times.
|
104766 | metrics.shownCorrect += correct ? 1 : 0; |
| 328 | } | ||
| 329 |
8/8✓ Branch 32 → 33 taken 96103 times.
✓ Branch 32 → 38 taken 8663 times.
✓ Branch 34 → 35 taken 96078 times.
✓ Branch 34 → 39 taken 25 times.
✓ Branch 37 → 38 taken 17612 times.
✓ Branch 37 → 39 taken 78466 times.
✓ Branch 40 → 41 taken 26275 times.
✓ Branch 40 → 42 taken 78491 times.
|
104766 | if (correct || (candidates.size() > 1 && candidates[1].folder == item.folder)) ++metrics.top2Correct; |
| 330 |
1/1✓ Branch 42 → 43 taken 104766 times.
|
104766 | bins.record(first.kind, first.strength, correct); |
| 331 | } | ||
| 332 |
1/1✓ Branch 43 → 44 taken 104889 times.
|
104889 | evidence.learn(item.mail, item.folder); |
| 333 | 104889 | } | |
| 334 | 266 | return metrics; | |
| 335 | 266 | } | |
| 336 |