data/MailSearchIndex.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | #pragma once | ||
| 2 | |||
| 3 | #include <QDateTime> | ||
| 4 | #include <QList> | ||
| 5 | #include <QSqlDatabase> | ||
| 6 | #include <QSqlQuery> | ||
| 7 | #include <QString> | ||
| 8 | #include <QStringList> | ||
| 9 | #include <atomic> | ||
| 10 | #include <functional> | ||
| 11 | #include <optional> | ||
| 12 | |||
| 13 | #include "data/MailSearchPage.h" | ||
| 14 | #include "data/MailSearchSummary.h" | ||
| 15 | |||
| 16 | // Full-text search (FTS5 trigram index over the search_documents | ||
| 17 | // projection), search queries and index maintenance of the mail cache | ||
| 18 | // (Sprint 91 S91.2.5, extracted from MailCache). Callers reach it as | ||
| 19 | // `cache.search()`; MailCache keeps its SearchResult / SearchFilter / | ||
| 20 | // SearchIndexDiagnostics names as type aliases. | ||
| 21 | // | ||
| 22 | // Plain class, no connection of its own: it uses the owning MailCache's | ||
| 23 | // QSqlDatabase and lastError string on that cache's thread (search readers | ||
| 24 | // open their own read-only MailCache). replaceSearchDocuments() and | ||
| 25 | // indexHeaderById() never start a transaction — MailCache::storeHeaderBatch, | ||
| 26 | // storeBodyPayload and the payload eviction call them inside their unit so | ||
| 27 | // the FTS projection stays atomic with the stored rows (rule 6). | ||
| 28 | // rebuildSearchIndex() and batchIndexForSearch() own their units as before. | ||
| 29 | class MailSearchIndex { | ||
| 30 | public: | ||
| 31 | struct Deps { | ||
| 32 | std::function<qint64(qint64 folderId, qint64 uid)> headerRowId; | ||
| 33 | }; | ||
| 34 | MailSearchIndex(QSqlDatabase &db, QString &lastError, Deps deps); | ||
| 35 | |||
| 36 | struct SearchResult { | ||
| 37 | qint64 folderId = 0; | ||
| 38 | qint64 uid = 0; | ||
| 39 | QString folderPath; // resolved folder path | ||
| 40 | QString subject; | ||
| 41 | QString from; | ||
| 42 | QString snippet; // highlighted match snippet | ||
| 43 | double rank = 0.0; // BM25 relevance score | ||
| 44 | }; | ||
| 45 | |||
| 46 | // Privacy-safe diagnostics for asserting exactly-one FTS ownership. The | ||
| 47 | // counters contain no message content and are scoped to this cache instance. | ||
| 48 | struct SearchIndexDiagnostics { | ||
| 49 | qint64 requests = 0; | ||
| 50 | qint64 replacements = 0; | ||
| 51 | qint64 unchangedSkipped = 0; | ||
| 52 | }; | ||
| 53 | |||
| 54 | 3 | SearchIndexDiagnostics searchIndexDiagnostics() const { | |
| 55 | 3 | return m_searchIndexDiagnostics; | |
| 56 | } | ||
| 57 | 3 | void resetSearchIndexDiagnostics() { m_searchIndexDiagnostics = {}; } | |
| 58 | // MailCache's header/body writers count their FTS requests and skips here. | ||
| 59 | 33769 | SearchIndexDiagnostics &diagnosticCounters() { return m_searchIndexDiagnostics; } | |
| 60 | |||
| 61 | // Search for emails matching a query string (FTS5 MATCH syntax). | ||
| 62 | // Returns results sorted by BM25 relevance, limited to maxResults. | ||
| 63 | QList<SearchResult> searchFts(const QString &query, int maxResults = 0, | ||
| 64 | int offset = 0) const; | ||
| 65 | |||
| 66 | // T-192: Optional filters for search. | ||
| 67 | // Sprint 59: extended with subject, tags and tri-state flag/attachment | ||
| 68 | // facets so the visual SearchPanel can express the full set of constraints. | ||
| 69 | struct SearchFilter { | ||
| 70 | // Tri-state for flag-like facets. "Any" means the facet adds no | ||
| 71 | // constraint; it maps 1:1 to the three states of the panel comboboxes. | ||
| 72 | enum class Tri { Any, Yes, No }; | ||
| 73 | |||
| 74 | // Sprint 60 (B1): zero, one or many folder patterns. OR-Semantik: a mail | ||
| 75 | // matches when its folder path contains ANY of the patterns (LIKE | ||
| 76 | // '%pattern%'). Empty list = no folder constraint. Replaces the single | ||
| 77 | // folderPattern of Sprint 59. | ||
| 78 | QStringList folderPatterns; // e.g. {"INBOX", "Archive"} — OR-matched | ||
| 79 | QDateTime dateFrom; // inclusive lower bound (empty = no lower bound) | ||
| 80 | QDateTime dateTo; // inclusive upper bound (empty = no upper bound) | ||
| 81 | QString fromFilter; // from address contains | ||
| 82 | QString toFilter; // To or Cc address contains (Cc: synced since v24) | ||
| 83 | QString listFilter; // List-Id contains (Sprint 95 S95.1.11) | ||
| 84 | |||
| 85 | // Sprint 59 facets: | ||
| 86 | QString subjectFilter; // folded subject substring (LIKE %X% on f.subject) | ||
| 87 | QStringList tags; // mail_labels: ALL listed labels must be present (AND) | ||
| 88 | Tri unread = Tri::Any; // Yes ⇒ Seen bit NOT set; No ⇒ Seen bit set | ||
| 89 | Tri flagged = Tri::Any; // Yes/No ⇒ Flagged bit set / not set | ||
| 90 | Tri answered = Tri::Any; // Yes/No ⇒ Answered bit set / not set | ||
| 91 | Tri hasAttachment = Tri::Any; // Yes/No ⇒ has_attachments = 1 / 0 | ||
| 92 | |||
| 93 | // Sprint 101 S101.1.5: the relative date word the range came from | ||
| 94 | // ("today", "yesterday", "week", "month", "year"); empty for absolute | ||
| 95 | // dates. Presentation only — the SQL uses dateFrom/dateTo — so a query | ||
| 96 | // written as "date:month" still reads "Last 30 days" after a round trip. | ||
| 97 | QString datePreset; | ||
| 98 | |||
| 99 | // Sprint 101 S101.1.8 (D-14): negated constraints ("-from:x", "-word"). | ||
| 100 | // A mail matches when it matches none of the listed values. | ||
| 101 | QStringList excludedFrom; // sender contains | ||
| 102 | QStringList excludedTo; // To or Cc contains | ||
| 103 | QStringList excludedSubjects; // folded subject contains | ||
| 104 | QStringList excludedFolders; // folder path contains | ||
| 105 | QStringList excludedLists; // List-Id contains | ||
| 106 | QStringList excludedTags; // has this label | ||
| 107 | QStringList excludedTerms; // free-text word (subject, sender, recipients, body) | ||
| 108 | |||
| 109 | // Sprint 103 D-21: about:<name> — mails linked to a contact (D-15). The | ||
| 110 | // read service resolves the name to the linked Message-IDs (canonical, | ||
| 111 | // at most MaxAboutMessageIds) before the query runs; an unresolved or | ||
| 112 | // empty list matches nothing. Only aboutFilter is part of the query text. | ||
| 113 | static constexpr int MaxAboutMessageIds = 2000; | ||
| 114 | QString aboutFilter; | ||
| 115 | QStringList aboutMessageIds; | ||
| 116 | |||
| 117 | 411 | bool hasExclusions() const { | |
| 118 |
2/4✓ Branch 5 → 6 taken 394 times.
✗ Branch 5 → 16 not taken.
✓ Branch 7 → 8 taken 394 times.
✗ Branch 7 → 16 not taken.
|
805 | return !excludedFrom.isEmpty() || !excludedTo.isEmpty() || !excludedSubjects.isEmpty() || |
| 119 |
8/8✓ Branch 3 → 4 taken 394 times.
✓ Branch 3 → 16 taken 17 times.
✓ Branch 9 → 10 taken 384 times.
✓ Branch 9 → 16 taken 10 times.
✓ Branch 11 → 12 taken 375 times.
✓ Branch 11 → 16 taken 9 times.
✓ Branch 13 → 14 taken 369 times.
✓ Branch 13 → 16 taken 6 times.
|
1174 | !excludedFolders.isEmpty() || !excludedLists.isEmpty() || !excludedTags.isEmpty() || |
| 120 |
2/2✓ Branch 15 → 16 taken 6 times.
✓ Branch 15 → 17 taken 363 times.
|
780 | !excludedTerms.isEmpty(); |
| 121 | } | ||
| 122 | |||
| 123 | // True when no facet constrains the search (every field at its neutral | ||
| 124 | // value). Used by the app path to distinguish "no search state" from | ||
| 125 | // "empty free text but active facets" (Sprint 59 A1). | ||
| 126 | 1033 | bool isEmpty() const { | |
| 127 |
2/2✓ Branch 5 → 6 taken 829 times.
✓ Branch 5 → 27 taken 144 times.
|
2006 | return folderPatterns.isEmpty() && !dateFrom.isValid() && |
| 128 |
10/10✓ Branch 7 → 8 taken 819 times.
✓ Branch 7 → 27 taken 10 times.
✓ Branch 9 → 10 taken 687 times.
✓ Branch 9 → 27 taken 132 times.
✓ Branch 11 → 12 taken 655 times.
✓ Branch 11 → 27 taken 32 times.
✓ Branch 13 → 14 taken 642 times.
✓ Branch 13 → 27 taken 13 times.
✓ Branch 15 → 16 taken 595 times.
✓ Branch 15 → 27 taken 47 times.
|
1471 | !dateTo.isValid() && fromFilter.isEmpty() && toFilter.isEmpty() && listFilter.isEmpty() && |
| 129 |
2/2✓ Branch 17 → 18 taken 561 times.
✓ Branch 17 → 27 taken 34 times.
|
1237 | subjectFilter.isEmpty() && tags.isEmpty() && |
| 130 |
4/4✓ Branch 18 → 19 taken 523 times.
✓ Branch 18 → 27 taken 38 times.
✓ Branch 19 → 20 taken 489 times.
✓ Branch 19 → 27 taken 34 times.
|
561 | unread == Tri::Any && flagged == Tri::Any && |
| 131 |
10/10✓ Branch 3 → 4 taken 973 times.
✓ Branch 3 → 27 taken 60 times.
✓ Branch 20 → 21 taken 449 times.
✓ Branch 20 → 27 taken 40 times.
✓ Branch 21 → 22 taken 417 times.
✓ Branch 21 → 27 taken 32 times.
✓ Branch 23 → 24 taken 410 times.
✓ Branch 23 → 27 taken 7 times.
✓ Branch 25 → 26 taken 363 times.
✓ Branch 25 → 27 taken 47 times.
|
2006 | answered == Tri::Any && hasAttachment == Tri::Any && aboutFilter.isEmpty() && !hasExclusions(); |
| 132 | } | ||
| 133 | |||
| 134 | 39 | bool operator==(const SearchFilter &o) const { | |
| 135 |
2/2✓ Branch 5 → 6 taken 35 times.
✓ Branch 5 → 41 taken 2 times.
|
76 | return folderPatterns == o.folderPatterns && dateFrom == o.dateFrom && |
| 136 |
6/6✓ Branch 7 → 8 taken 33 times.
✓ Branch 7 → 41 taken 2 times.
✓ Branch 9 → 10 taken 31 times.
✓ Branch 9 → 41 taken 2 times.
✓ Branch 11 → 12 taken 29 times.
✓ Branch 11 → 41 taken 2 times.
|
66 | dateTo == o.dateTo && fromFilter == o.fromFilter && |
| 137 |
3/4✓ Branch 13 → 14 taken 29 times.
✗ Branch 13 → 41 not taken.
✓ Branch 15 → 16 taken 27 times.
✓ Branch 15 → 41 taken 2 times.
|
60 | toFilter == o.toFilter && listFilter == o.listFilter && subjectFilter == o.subjectFilter && |
| 138 |
6/6✓ Branch 17 → 18 taken 25 times.
✓ Branch 17 → 41 taken 2 times.
✓ Branch 18 → 19 taken 21 times.
✓ Branch 18 → 41 taken 4 times.
✓ Branch 19 → 20 taken 17 times.
✓ Branch 19 → 41 taken 4 times.
|
27 | tags == o.tags && unread == o.unread && flagged == o.flagged && |
| 139 |
5/6✓ Branch 20 → 21 taken 13 times.
✓ Branch 20 → 41 taken 4 times.
✓ Branch 21 → 22 taken 9 times.
✓ Branch 21 → 41 taken 4 times.
✓ Branch 23 → 24 taken 9 times.
✗ Branch 23 → 41 not taken.
|
26 | answered == o.answered && hasAttachment == o.hasAttachment && |
| 140 |
1/2✓ Branch 25 → 26 taken 9 times.
✗ Branch 25 → 41 not taken.
|
18 | datePreset == o.datePreset && excludedFrom == o.excludedFrom && |
| 141 |
2/4✓ Branch 27 → 28 taken 9 times.
✗ Branch 27 → 41 not taken.
✓ Branch 29 → 30 taken 9 times.
✗ Branch 29 → 41 not taken.
|
9 | excludedTo == o.excludedTo && excludedSubjects == o.excludedSubjects && |
| 142 |
2/4✓ Branch 31 → 32 taken 9 times.
✗ Branch 31 → 41 not taken.
✓ Branch 33 → 34 taken 9 times.
✗ Branch 33 → 41 not taken.
|
9 | excludedFolders == o.excludedFolders && excludedLists == o.excludedLists && |
| 143 |
5/8✓ Branch 3 → 4 taken 37 times.
✓ Branch 3 → 41 taken 2 times.
✓ Branch 35 → 36 taken 9 times.
✗ Branch 35 → 41 not taken.
✓ Branch 37 → 38 taken 9 times.
✗ Branch 37 → 41 not taken.
✓ Branch 39 → 40 taken 9 times.
✗ Branch 39 → 41 not taken.
|
76 | excludedTags == o.excludedTags && excludedTerms == o.excludedTerms && aboutFilter == o.aboutFilter; |
| 144 | } | ||
| 145 | 15 | bool operator!=(const SearchFilter &o) const { return !(*this == o); } | |
| 146 | 1735 | bool fitsReadLimits(const QString &query) const { | |
| 147 | 1735 | const qsizetype excludedCount = excludedFrom.size() + excludedTo.size() + excludedSubjects.size() + | |
| 148 | 1735 | excludedFolders.size() + excludedLists.size() + excludedTags.size() + excludedTerms.size(); | |
| 149 |
10/12✓ Branch 10 → 11 taken 1734 times.
✓ Branch 10 → 18 taken 1 time.
✓ Branch 12 → 13 taken 1732 times.
✓ Branch 12 → 18 taken 2 times.
✓ Branch 14 → 15 taken 1732 times.
✗ Branch 14 → 18 not taken.
✓ Branch 15 → 16 taken 1731 times.
✓ Branch 15 → 18 taken 1 time.
✗ Branch 17 → 18 not taken.
✓ Branch 17 → 19 taken 1731 times.
✓ Branch 20 → 21 taken 4 times.
✓ Branch 20 → 22 taken 1731 times.
|
3466 | if (query.size() > 4096 || tags.size() > 128 || folderPatterns.size() > 128 || excludedCount > 128 || |
| 150 | 1731 | aboutMessageIds.size() > MaxAboutMessageIds) | |
| 151 | 4 | return false; | |
| 152 | 1731 | qint64 characters = query.size() + qint64(fromFilter.size()) + toFilter.size() + listFilter.size() + | |
| 153 | 1731 | subjectFilter.size() + aboutFilter.size(); | |
| 154 |
2/2✓ Branch 34 → 30 taken 40 times.
✓ Branch 34 → 35 taken 1731 times.
|
1771 | for (const auto &tag : tags) characters += tag.size(); |
| 155 |
2/2✓ Branch 41 → 37 taken 104 times.
✓ Branch 41 → 42 taken 1731 times.
|
1835 | for (const auto &folder : folderPatterns) characters += folder.size(); |
| 156 | 13848 | for (const auto *list : {&excludedFrom, &excludedTo, &excludedSubjects, &excludedFolders, | |
| 157 |
2/2✓ Branch 53 → 45 taken 12117 times.
✓ Branch 53 → 54 taken 1731 times.
|
15579 | &excludedLists, &excludedTags, &excludedTerms}) |
| 158 |
2/2✓ Branch 51 → 47 taken 151 times.
✓ Branch 51 → 52 taken 12117 times.
|
12268 | for (const auto &value : *list) characters += value.size(); |
| 159 | 1731 | return characters <= 16384; | |
| 160 | } | ||
| 161 | }; | ||
| 162 | QList<SearchResult> searchFts(const QString &query, const SearchFilter &filter, | ||
| 163 | int maxResults = 0, int offset = 0, | ||
| 164 | const std::optional<QString> &accountId = std::nullopt, | ||
| 165 | QString *error = nullptr) const; | ||
| 166 | MailSearchPage searchPage(const QString &query, const SearchFilter &filter, | ||
| 167 | const std::optional<QString> &accountId, int limit, | ||
| 168 | int offset, const std::atomic_bool &cancelled) const; | ||
| 169 | // Sprint 101 S101.1.3: counts, days, hours and top facet values of every | ||
| 170 | // hit, on the same WHERE as searchPage() (MailSearchSummary.cpp). Bounded | ||
| 171 | // by MailSearchSummary::TimeLimitMs and `cancelled`; a limit reached | ||
| 172 | // returns Limited with the distinct mails counted so far in `total`. | ||
| 173 | // A non-empty `allowedFolders` counts hits in those folder paths only | ||
| 174 | // (the list's folder admission). | ||
| 175 | MailSearchSummary searchSummary(const QString &query, const SearchFilter &filter, | ||
| 176 | const std::optional<QString> &accountId, | ||
| 177 | MailSearchSummary::Detail detail, | ||
| 178 | const std::atomic_bool &cancelled, | ||
| 179 | int timeLimitMs = MailSearchSummary::TimeLimitMs, | ||
| 180 | const QSet<QString> &allowedFolders = {}) const; | ||
| 181 | |||
| 182 | // Sprint 59 (F3): distinct IMAP keywords/labels known to the cache, sorted | ||
| 183 | // case-insensitively. Used to populate tag suggestions in the search UI. | ||
| 184 | QStringList knownLabels() const; | ||
| 185 | |||
| 186 | // Index a single email for FTS5 (called after storeHeaders/storeBody). | ||
| 187 | void indexForSearch(qint64 folderId, qint64 uid); | ||
| 188 | void batchIndexForSearch(qint64 folderId, const QList<qint64> &uids); | ||
| 189 | bool searchIndexEmpty() const; | ||
| 190 | bool searchIndexNeedsRebuild() const; | ||
| 191 | |||
| 192 | // Rebuild the entire FTS5 index from existing cached data. | ||
| 193 | // This is a heavyweight operation, intended for initial population or repair. | ||
| 194 | bool rebuildSearchIndex(); | ||
| 195 | // Rows per committed keyset page when the rebuild repairs a | ||
| 196 | // search_documents projection that lost rows. | ||
| 197 | static constexpr int kRepairBatchRows = 1000; | ||
| 198 | |||
| 199 | // Fold text for the FTS index: lowercase + strip diacritics (Müller→muller, | ||
| 200 | // Café→cafe, ß→ss). Applied to BOTH indexed text and queries so the trigram | ||
| 201 | // tokenizer (no native remove_diacritics before SQLite 3.45) is accent-blind. | ||
| 202 | // Public + static so it can be unit-tested in isolation. | ||
| 203 | static QString foldForSearch(const QString &text); | ||
| 204 | // SEC-2026-07-21-22: Escape LIKE wildcards (% _ \) in user-supplied terms. | ||
| 205 | static QString escapeLikePattern(const QString &raw); | ||
| 206 | |||
| 207 | // Replace a header's FTS row from current header/body data. The caller must | ||
| 208 | // provide a transaction when this is used concurrently with other writers. | ||
| 209 | bool indexHeaderById(qint64 rowId); | ||
| 210 | |||
| 211 | struct SearchDocument { | ||
| 212 | qint64 rowId = 0; | ||
| 213 | QString subject; | ||
| 214 | QString from; | ||
| 215 | QString to; | ||
| 216 | QString textPlain; | ||
| 217 | QString textHtml; | ||
| 218 | }; | ||
| 219 | std::optional<QList<SearchDocument>> searchDocumentsByIds(const QList<qint64> &rowIds); | ||
| 220 | bool replaceSearchDocuments(const QList<SearchDocument> &documents, | ||
| 221 | bool deleteExisting = true); | ||
| 222 | |||
| 223 | private: | ||
| 224 | // rebuildSearchIndex() steps (S91.3.9). | ||
| 225 | bool markRebuildInProgress(); | ||
| 226 | bool repairSearchDocuments(int headerCount, int documentCount, int &repairedCount); | ||
| 227 | // searchFtsImpl() steps (S91.3.9). | ||
| 228 | static void splitSearchTerms(const QString &query, QStringList &matchTerms, | ||
| 229 | QStringList &likeTerms); | ||
| 230 | static QStringList searchConditions(const SearchFilter &filter, bool accountScoped, | ||
| 231 | bool useMatch, const QStringList &likeTerms, | ||
| 232 | const QStringList &folderPatterns); | ||
| 233 | static void splitExcludedTerms(const SearchFilter &filter, QStringList &matchTerms, | ||
| 234 | QStringList &likeTerms); | ||
| 235 | static QStringList exclusionConditions(const SearchFilter &filter); | ||
| 236 | static void bindExclusions(QSqlQuery &q, const SearchFilter &filter); | ||
| 237 | static QString searchPageProjection(QString sql); | ||
| 238 | static void bindSearchParameters(QSqlQuery &q, const SearchFilter &filter, | ||
| 239 | const std::optional<QString> &accountId, | ||
| 240 | const QString &matchQuery, const QStringList &likeTerms, | ||
| 241 | const QStringList &folderPatterns, int maxResults, | ||
| 242 | int offset); | ||
| 243 | QList<SearchResult> searchFtsImpl(const QString &query, const SearchFilter &filter, | ||
| 244 | int maxResults, int offset, const std::optional<QString> &accountId, | ||
| 245 | QString *error, MailSearchPage *page) const; | ||
| 246 | QSqlDatabase &m_db; // owned by MailCache | ||
| 247 | QString &m_lastError; // MailCache::lastError() | ||
| 248 | Deps m_deps; | ||
| 249 | SearchIndexDiagnostics m_searchIndexDiagnostics; | ||
| 250 | }; | ||
| 251 |