From fb721c4981fe860484abd24eab2dd1edb48e6b75 Mon Sep 17 00:00:00 2001 From: Zhang Sheng Date: Fri, 17 Jul 2026 11:59:22 +0800 Subject: [PATCH] fix(search): return plain verbose snippets for text search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use plain snippets for content and OCR verbose output. 为内容和 OCR 的详细输出使用纯文本片段。 Log: 修复详细搜索结果片段输出 Influence: content/ocr 的 -v 输出改为纯文本，并新增 snippetOffset。 --- .../tst_content_retriever.cpp | 27 ++++++++++ .../dfm-search-tests/tst_textsearch_api.cpp | 15 ++++++ .../dfm-search-client/output/json_output.cpp | 29 +++++++++- .../dfm-search-client/output/text_output.cpp | 30 ++++++++++- .../contentstrategies/indexedstrategy.cpp | 8 +++ .../ocrtextstrategies/indexedstrategy.cpp | 8 +++ .../utils/contenthighlighter.cpp | 54 +++++++++++++++++++ .../dfm-search-lib/utils/contenthighlighter.h | 23 ++++++++ 8 files changed, 190 insertions(+), 4 deletions(-) diff --git a/autotests/dfm-search-tests/tst_content_retriever.cpp b/autotests/dfm-search-tests/tst_content_retriever.cpp index 480821ca..2525a548 100644 --- a/autotests/dfm-search-tests/tst_content_retriever.cpp +++ b/autotests/dfm-search-tests/tst_content_retriever.cpp @@ -99,6 +99,7 @@ private Q_SLOTS: void fetchPreview_offsetBeyondContent(); void fetchPreview_unlimitedNoKeyword(); void previewSnippet_basic(); + void plainSnippet_basic(); }; void tst_ContentRetriever::fetchContent_single() @@ -407,6 +408,32 @@ void tst_ContentRetriever::previewSnippet_basic() QCOMPARE(kwOff, 6); } +void tst_ContentRetriever::plainSnippet_basic() +{ + using DFMSEARCH::ContentHighlighter::PlainSnippetResult; + using DFMSEARCH::ContentHighlighter::plainSnippet; + + const QString content = QStringLiteral("0123456789abcdefghijKEYWORDklmnopqrstuvwxyz"); + + { + const PlainSnippetResult result = plainSnippet({ "keyword" }, content, 20, 20); + QCOMPARE(result.content, QString("abcdefghijKEYWORDklm")); + QCOMPARE(result.snippetOffset, 10); + } + + { + const PlainSnippetResult result = plainSnippet({ "keyword", "0123" }, content, 20, 20); + QCOMPARE(result.content, QString("0123456789abcdefghij")); + QCOMPARE(result.snippetOffset, 0); + } + + { + const PlainSnippetResult result = plainSnippet({ "missing" }, content, 20, 20); + QVERIFY(result.content.isEmpty()); + QCOMPARE(result.snippetOffset, -1); + } +} + QObject *create_tst_ContentRetriever() { return new tst_ContentRetriever(); diff --git a/autotests/dfm-search-tests/tst_textsearch_api.cpp b/autotests/dfm-search-tests/tst_textsearch_api.cpp index 92dec9e9..bc398da2 100644 --- a/autotests/dfm-search-tests/tst_textsearch_api.cpp +++ b/autotests/dfm-search-tests/tst_textsearch_api.cpp @@ -31,6 +31,7 @@ private Q_SLOTS: void textSearchResult_isHidden(); void textSearchResult_modifyTimestamp(); void textSearchResult_birthTimestamp(); + void textSearchResult_plainSnippetAttributes(); // ContentOptionsAPI tests void contentOptions_inheritance(); @@ -184,6 +185,20 @@ void tst_TextSearchAPI::textSearchResult_birthTimestamp() QCOMPARE(api.birthTimestamp(), 0); } +void tst_TextSearchAPI::textSearchResult_plainSnippetAttributes() +{ + SearchResult result("/test/path"); + + QVERIFY(!result.hasCustomAttribute("plainContentMatch")); + QVERIFY(!result.hasCustomAttribute("snippetOffset")); + + result.setCustomAttribute("plainContentMatch", QString("plain snippet")); + result.setCustomAttribute("snippetOffset", 42); + + QCOMPARE(result.customAttribute("plainContentMatch").toString(), QString("plain snippet")); + QCOMPARE(result.customAttribute("snippetOffset").toInt(), 42); +} + // ==================== ContentOptionsAPI Tests ==================== void tst_TextSearchAPI::contentOptions_inheritance() diff --git a/src/dfm-search/dfm-search-client/output/json_output.cpp b/src/dfm-search/dfm-search-client/output/json_output.cpp index 35fbeaa8..582b2205 100644 --- a/src/dfm-search/dfm-search-client/output/json_output.cpp +++ b/src/dfm-search/dfm-search-client/output/json_output.cpp @@ -8,11 +8,34 @@ #include #include +#include #include using namespace dfmsearch; using namespace std; +namespace { + +QString plainContentMatch(const SearchResult &result) +{ + if (result.hasCustomAttribute("plainContentMatch")) { + return result.customAttribute("plainContentMatch").toString(); + } + + QString content = result.customAttribute("highlightedContent").toString(); + static const QRegularExpression kHtmlTagPattern(QStringLiteral("<[^>]+>")); + return content.remove(kHtmlTagPattern); +} + +int snippetOffset(const SearchResult &result) +{ + return result.hasCustomAttribute("snippetOffset") + ? result.customAttribute("snippetOffset").toInt() + : -1; +} + +} // namespace + void JsonOutput::setSearchContext(const QString &keyword, const QString &searchPath, SearchType searchType, SearchMethod searchMethod) { @@ -204,7 +227,8 @@ QJsonValue JsonOutput::resultToJson(const SearchResult &result) obj["path"] = result.path(); ContentResultAPI resultAPI(const_cast(result)); - obj["contentMatch"] = resultAPI.highlightedContent(); + obj["contentMatch"] = plainContentMatch(result); + obj["snippetOffset"] = snippetOffset(result); QString filename = resultAPI.filename(); if (!filename.isEmpty()) { @@ -245,7 +269,8 @@ QJsonValue JsonOutput::resultToJson(const SearchResult &result) OcrTextResultAPI resultAPI(const_cast(result)); - obj["contentMatch"] = resultAPI.highlightedContent(); + obj["contentMatch"] = plainContentMatch(result); + obj["snippetOffset"] = snippetOffset(result); QString ocrContent = resultAPI.ocrContent(); if (!ocrContent.isEmpty()) { diff --git a/src/dfm-search/dfm-search-client/output/text_output.cpp b/src/dfm-search/dfm-search-client/output/text_output.cpp index 0edc37c9..7d43eb7f 100644 --- a/src/dfm-search/dfm-search-client/output/text_output.cpp +++ b/src/dfm-search/dfm-search-client/output/text_output.cpp @@ -7,11 +7,35 @@ #include #include +#include + #include using namespace dfmsearch; using namespace std; +namespace { + +QString plainContentMatch(const SearchResult &result) +{ + if (result.hasCustomAttribute("plainContentMatch")) { + return result.customAttribute("plainContentMatch").toString(); + } + + QString content = result.customAttribute("highlightedContent").toString(); + static const QRegularExpression kHtmlTagPattern(QStringLiteral("<[^>]+>")); + return content.remove(kHtmlTagPattern); +} + +int snippetOffset(const SearchResult &result) +{ + return result.hasCustomAttribute("snippetOffset") + ? result.customAttribute("snippetOffset").toInt() + : -1; +} + +} // namespace + void TextOutput::setSearchContext(const QString &keyword, const QString &searchPath, SearchType searchType, SearchMethod searchMethod) { @@ -128,7 +152,8 @@ void TextOutput::printSearchResult(const SearchResult &result) } else if (m_searchType == SearchType::Content) { ContentResultAPI resultAPI(const_cast(result)); - std::cout << " Content match: " << resultAPI.highlightedContent().toStdString() << std::endl; + std::cout << " Content match: " << plainContentMatch(result).toStdString() << std::endl; + std::cout << " Snippet offset: " << snippetOffset(result) << std::endl; // 文件名 QString filename = resultAPI.filename(); @@ -161,7 +186,8 @@ void TextOutput::printSearchResult(const SearchResult &result) } else if (m_searchType == SearchType::Ocr) { OcrTextResultAPI resultAPI(const_cast(result)); - std::cout << " Content match: " << resultAPI.highlightedContent().toStdString() << std::endl; + std::cout << " Content match: " << plainContentMatch(result).toStdString() << std::endl; + std::cout << " Snippet offset: " << snippetOffset(result) << std::endl; QString ocrContent = resultAPI.ocrContent(); if (!ocrContent.isEmpty()) { diff --git a/src/dfm-search/dfm-search-lib/contentsearch/contentstrategies/indexedstrategy.cpp b/src/dfm-search/dfm-search-lib/contentsearch/contentstrategies/indexedstrategy.cpp index b6636d12..a940cb6f 100644 --- a/src/dfm-search/dfm-search-lib/contentsearch/contentstrategies/indexedstrategy.cpp +++ b/src/dfm-search/dfm-search-lib/contentsearch/contentstrategies/indexedstrategy.cpp @@ -405,14 +405,22 @@ void ContentIndexedStrategy::processSearchResults(const Lucene::IndexSearcherPtr SearchResult result(QString::fromStdWString(pathField)); ContentResultAPI resultApi(result); + result.setCustomAttribute("plainContentMatch", QString()); + result.setCustomAttribute("snippetOffset", -1); if (enableRetrieval) { try { Lucene::String contentField = doc->get(LuceneFieldNames::Content::kContents); if (!contentField.empty()) { const QString content = QString::fromStdWString(contentField); + // Fix: keep a dedicated plain-text snippet for CLI verbose output + // instead of reusing HTML-oriented highlightedContent. + const ContentHighlighter::PlainSnippetResult plainSnippet = ContentHighlighter::plainSnippet( + m_keywords, content, previewLen, previewLen); const QString highlightedContent = ContentHighlighter::customHighlight(m_keywords, content, previewLen, enableHTML); resultApi.setHighlightedContent(highlightedContent); + result.setCustomAttribute("plainContentMatch", plainSnippet.content); + result.setCustomAttribute("snippetOffset", plainSnippet.snippetOffset); } } catch (const Lucene::LuceneException &e) { qWarning() << "Exception retrieving content field:" << QString::fromStdWString(e.getError()); diff --git a/src/dfm-search/dfm-search-lib/ocrtextsearch/ocrtextstrategies/indexedstrategy.cpp b/src/dfm-search/dfm-search-lib/ocrtextsearch/ocrtextstrategies/indexedstrategy.cpp index 27b5a872..4b62082c 100644 --- a/src/dfm-search/dfm-search-lib/ocrtextsearch/ocrtextstrategies/indexedstrategy.cpp +++ b/src/dfm-search/dfm-search-lib/ocrtextsearch/ocrtextstrategies/indexedstrategy.cpp @@ -404,6 +404,8 @@ void OcrTextIndexedStrategy::processSearchResults(const Lucene::IndexSearcherPtr // 设置 OCR 内容结果 OcrTextResultAPI resultApi(result); + result.setCustomAttribute("plainContentMatch", QString()); + result.setCustomAttribute("snippetOffset", -1); // 使用ContentHighlighter命名空间进行高亮 if (enableRetrieval) { @@ -411,12 +413,18 @@ void OcrTextIndexedStrategy::processSearchResults(const Lucene::IndexSearcherPtr Lucene::String ocrContentField = doc->get(LuceneFieldNames::OcrText::kOcrContents); if (!ocrContentField.empty()) { const QString content = QString::fromStdWString(ocrContentField); + // Fix: keep a dedicated plain-text snippet for CLI verbose output + // instead of reusing HTML-oriented highlightedContent. + const ContentHighlighter::PlainSnippetResult plainSnippet = ContentHighlighter::plainSnippet( + m_keywords, content, previewLen, previewLen); // 设置原始 OCR 内容 resultApi.setOcrContent(content); // 设置高亮内容 const QString highlightedContent = ContentHighlighter::customHighlight( m_keywords, content, previewLen, enableHTML); resultApi.setHighlightedContent(highlightedContent); + result.setCustomAttribute("plainContentMatch", plainSnippet.content); + result.setCustomAttribute("snippetOffset", plainSnippet.snippetOffset); } } catch (const Lucene::LuceneException &e) { qWarning() << "Exception retrieving OCR content field:" << QString::fromStdWString(e.getError()); diff --git a/src/dfm-search/dfm-search-lib/utils/contenthighlighter.cpp b/src/dfm-search/dfm-search-lib/utils/contenthighlighter.cpp index 43caf75f..3c606d76 100644 --- a/src/dfm-search/dfm-search-lib/utils/contenthighlighter.cpp +++ b/src/dfm-search/dfm-search-lib/utils/contenthighlighter.cpp @@ -29,6 +29,15 @@ struct KeywordMatch QString keyword; }; +int positioningBeforeLength(int maxLength, int positioningMaxLength) +{ + const int effectivePositioningLength = qMax(30, positioningMaxLength); + if (maxLength > 0) { + return qMin(effectivePositioningLength / 2, maxLength / 2); + } + return effectivePositioningLength / 2; +} + /** * @brief 检查给定位置是否是段落的开头 * @param content 原始文本内容 @@ -153,6 +162,24 @@ KeywordMatch findFirstKeywordMatch(const QString &content, const QStringList &ke } return match; } + +KeywordMatch findEarliestKeywordMatch(const QString &content, const QStringList &keywords) +{ + KeywordMatch match = { -1, 0, QString() }; + for (const QString &keyword : keywords) { + if (keyword.isEmpty()) { + continue; + } + + const int pos = content.indexOf(keyword, 0, Qt::CaseInsensitive); + if (pos != -1 && (match.position == -1 || pos < match.position)) { + match.position = pos; + match.length = keyword.length(); + match.keyword = keyword; + } + } + return match; +} } // namespace QString customHighlight(const QStringList &keywords, const QString &content, int maxLength, bool enableHtml, int positioningMaxLength) @@ -242,6 +269,33 @@ QString customHighlight(const QStringList &keywords, const QString &content, int return resultSnippet; } +PlainSnippetResult plainSnippet(const QStringList &keywords, const QString &content, + int maxLength, int positioningMaxLength) +{ + PlainSnippetResult result; + if (content.isEmpty() || keywords.isEmpty()) { + return result; + } + + const KeywordMatch match = findEarliestKeywordMatch(content, keywords); + if (match.position == -1) { + return result; + } + + int snippetStart = qMax(0, match.position - positioningBeforeLength(maxLength, positioningMaxLength)); + if (maxLength > 0 && content.length() - snippetStart < maxLength) { + snippetStart = qMax(0, content.length() - maxLength); + } + + const int snippetLength = maxLength > 0 + ? qMin(maxLength, content.length() - snippetStart) + : content.length() - snippetStart; + + result.content = content.mid(snippetStart, snippetLength); + result.snippetOffset = snippetStart; + return result; +} + QString previewSnippet(const QString &content, int offset, int maxLength, const QString &keyword, int *keywordOffset) { diff --git a/src/dfm-search/dfm-search-lib/utils/contenthighlighter.h b/src/dfm-search/dfm-search-lib/utils/contenthighlighter.h index 89ef5d7f..08829a13 100644 --- a/src/dfm-search/dfm-search-lib/utils/contenthighlighter.h +++ b/src/dfm-search/dfm-search-lib/utils/contenthighlighter.h @@ -15,6 +15,12 @@ DFM_SEARCH_BEGIN_NS namespace ContentHighlighter { +struct PlainSnippetResult +{ + QString content; + int snippetOffset = -1; +}; + /** * @brief Extracts and highlights the most relevant text snippet from content based on the provided keywords. * @@ -40,6 +46,23 @@ namespace ContentHighlighter { */ QString customHighlight(const QStringList &keywords, const QString &content, int maxLength, bool enableHtml, int positioningMaxLength = 30); +/** + * @brief Extracts a plain-text snippet for CLI verbose output. + * + * Unlike customHighlight(), this helper preserves the original content bytes + * in the returned snippet: it does not simplify whitespace, insert HTML tags, + * or prepend ellipsis. The snippet start is returned so callers can expose the + * offset in the original full text. + * + * @param keywords Search keywords used to anchor the snippet. + * @param content Original document content. + * @param maxLength Maximum length of the returned snippet. <= 0 means until EOF. + * @param positioningMaxLength Window used to position the first content match. + * @return PlainSnippetResult with snippet text and snippetOffset, or empty/-1 if no content match exists. + */ +PlainSnippetResult plainSnippet(const QStringList &keywords, const QString &content, + int maxLength, int positioningMaxLength = 30); + /** * @brief 精确预览：从 offset 位置截取内容，或从 offset 搜索 keyword 后从匹配位置截取 *