Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions autotests/dfm-search-tests/tst_content_retriever.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ private Q_SLOTS:
void fetchPreview_offsetBeyondContent();
void fetchPreview_unlimitedNoKeyword();
void previewSnippet_basic();
void plainSnippet_basic();
};

void tst_ContentRetriever::fetchContent_single()
Expand Down Expand Up @@ -407,6 +408,32 @@ void tst_ContentRetriever::previewSnippet_basic()
QCOMPARE(kwOff, 6);
}

void tst_ContentRetriever::plainSnippet_basic()
{
using DFMSEARCH::ContentHighlighter::PlainSnippetResult;
using DFMSEARCH::ContentHighlighter::plainSnippet;

const QString content = QStringLiteral("0123456789abcdefghijKEYWORDklmnopqrstuvwxyz");

{
const PlainSnippetResult result = plainSnippet({ "keyword" }, content, 20, 20);
QCOMPARE(result.content, QString("abcdefghijKEYWORDklm"));
QCOMPARE(result.snippetOffset, 10);
}

{
const PlainSnippetResult result = plainSnippet({ "keyword", "0123" }, content, 20, 20);
QCOMPARE(result.content, QString("0123456789abcdefghij"));
QCOMPARE(result.snippetOffset, 0);
}

{
const PlainSnippetResult result = plainSnippet({ "missing" }, content, 20, 20);
QVERIFY(result.content.isEmpty());
QCOMPARE(result.snippetOffset, -1);
}
}

QObject *create_tst_ContentRetriever()
{
return new tst_ContentRetriever();
Expand Down
15 changes: 15 additions & 0 deletions autotests/dfm-search-tests/tst_textsearch_api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ private Q_SLOTS:
void textSearchResult_isHidden();
void textSearchResult_modifyTimestamp();
void textSearchResult_birthTimestamp();
void textSearchResult_plainSnippetAttributes();

// ContentOptionsAPI tests
void contentOptions_inheritance();
Expand Down Expand Up @@ -184,6 +185,20 @@ void tst_TextSearchAPI::textSearchResult_birthTimestamp()
QCOMPARE(api.birthTimestamp(), 0);
}

void tst_TextSearchAPI::textSearchResult_plainSnippetAttributes()
{
SearchResult result("/test/path");

QVERIFY(!result.hasCustomAttribute("plainContentMatch"));
QVERIFY(!result.hasCustomAttribute("snippetOffset"));

result.setCustomAttribute("plainContentMatch", QString("plain snippet"));
result.setCustomAttribute("snippetOffset", 42);

QCOMPARE(result.customAttribute("plainContentMatch").toString(), QString("plain snippet"));
QCOMPARE(result.customAttribute("snippetOffset").toInt(), 42);
}

// ==================== ContentOptionsAPI Tests ====================

void tst_TextSearchAPI::contentOptions_inheritance()
Expand Down
29 changes: 27 additions & 2 deletions src/dfm-search/dfm-search-client/output/json_output.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,37 @@
#include "json_output.h"
#include <dfm-search/filenamesearchapi.h>
#include <dfm-search/contentsearchapi.h>
#include <dfm-search/ocrtextsearchapi.h>

Check warning on line 8 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <dfm-search/ocrtextsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 8 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <dfm-search/ocrtextsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.

#include <QJsonDocument>

Check warning on line 10 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <QJsonDocument> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 10 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <QJsonDocument> not found. Please note: Cppcheck does not need standard library headers to get proper results.
#include <QRegularExpression>

Check warning on line 11 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <QRegularExpression> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 11 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <QRegularExpression> not found. Please note: Cppcheck does not need standard library headers to get proper results.
#include <iostream>

Check warning on line 12 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <iostream> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 12 in src/dfm-search/dfm-search-client/output/json_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <iostream> not found. Please note: Cppcheck does not need standard library headers to get proper results.

using namespace dfmsearch;
using namespace std;

namespace {

QString plainContentMatch(const SearchResult &result)
{
if (result.hasCustomAttribute("plainContentMatch")) {
return result.customAttribute("plainContentMatch").toString();
}

QString content = result.customAttribute("highlightedContent").toString();
static const QRegularExpression kHtmlTagPattern(QStringLiteral("<[^>]+>"));
return content.remove(kHtmlTagPattern);
}

int snippetOffset(const SearchResult &result)
{
return result.hasCustomAttribute("snippetOffset")
? result.customAttribute("snippetOffset").toInt()
: -1;
}

} // namespace

void JsonOutput::setSearchContext(const QString &keyword, const QString &searchPath,
SearchType searchType, SearchMethod searchMethod)
{
Expand Down Expand Up @@ -204,7 +227,8 @@
obj["path"] = result.path();

ContentResultAPI resultAPI(const_cast<SearchResult &>(result));
obj["contentMatch"] = resultAPI.highlightedContent();
obj["contentMatch"] = plainContentMatch(result);
obj["snippetOffset"] = snippetOffset(result);

QString filename = resultAPI.filename();
if (!filename.isEmpty()) {
Expand Down Expand Up @@ -245,7 +269,8 @@

OcrTextResultAPI resultAPI(const_cast<SearchResult &>(result));

obj["contentMatch"] = resultAPI.highlightedContent();
obj["contentMatch"] = plainContentMatch(result);
obj["snippetOffset"] = snippetOffset(result);

QString ocrContent = resultAPI.ocrContent();
if (!ocrContent.isEmpty()) {
Expand Down
30 changes: 28 additions & 2 deletions src/dfm-search/dfm-search-client/output/text_output.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,38 @@

#include "text_output.h"
#include <dfm-search/filenamesearchapi.h>
#include <dfm-search/contentsearchapi.h>

Check warning on line 7 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <dfm-search/contentsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 7 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <dfm-search/contentsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.
#include <dfm-search/ocrtextsearchapi.h>

Check warning on line 8 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <dfm-search/ocrtextsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 8 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <dfm-search/ocrtextsearchapi.h> not found. Please note: Cppcheck does not need standard library headers to get proper results.

#include <QRegularExpression>

Check warning on line 10 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <QRegularExpression> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 10 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <QRegularExpression> not found. Please note: Cppcheck does not need standard library headers to get proper results.

#include <iostream>

Check warning on line 12 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / cppcheck

Include file: <iostream> not found. Please note: Cppcheck does not need standard library headers to get proper results.

Check warning on line 12 in src/dfm-search/dfm-search-client/output/text_output.cpp

View workflow job for this annotation

GitHub Actions / static-check / static-check

Include file: <iostream> not found. Please note: Cppcheck does not need standard library headers to get proper results.

using namespace dfmsearch;
using namespace std;

namespace {

QString plainContentMatch(const SearchResult &result)
{
if (result.hasCustomAttribute("plainContentMatch")) {
return result.customAttribute("plainContentMatch").toString();
}

QString content = result.customAttribute("highlightedContent").toString();
static const QRegularExpression kHtmlTagPattern(QStringLiteral("<[^>]+>"));
return content.remove(kHtmlTagPattern);
}

int snippetOffset(const SearchResult &result)
{
return result.hasCustomAttribute("snippetOffset")
? result.customAttribute("snippetOffset").toInt()
: -1;
}

} // namespace

void TextOutput::setSearchContext(const QString &keyword, const QString &searchPath,
SearchType searchType, SearchMethod searchMethod)
{
Expand Down Expand Up @@ -128,7 +152,8 @@
} else if (m_searchType == SearchType::Content) {
ContentResultAPI resultAPI(const_cast<SearchResult &>(result));

std::cout << " Content match: " << resultAPI.highlightedContent().toStdString() << std::endl;
std::cout << " Content match: " << plainContentMatch(result).toStdString() << std::endl;
std::cout << " Snippet offset: " << snippetOffset(result) << std::endl;

// 文件名
QString filename = resultAPI.filename();
Expand Down Expand Up @@ -161,7 +186,8 @@
} else if (m_searchType == SearchType::Ocr) {
OcrTextResultAPI resultAPI(const_cast<SearchResult &>(result));

std::cout << " Content match: " << resultAPI.highlightedContent().toStdString() << std::endl;
std::cout << " Content match: " << plainContentMatch(result).toStdString() << std::endl;
std::cout << " Snippet offset: " << snippetOffset(result) << std::endl;

QString ocrContent = resultAPI.ocrContent();
if (!ocrContent.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -405,14 +405,22 @@ void ContentIndexedStrategy::processSearchResults(const Lucene::IndexSearcherPtr

SearchResult result(QString::fromStdWString(pathField));
ContentResultAPI resultApi(result);
result.setCustomAttribute("plainContentMatch", QString());
result.setCustomAttribute("snippetOffset", -1);

if (enableRetrieval) {
try {
Lucene::String contentField = doc->get(LuceneFieldNames::Content::kContents);
if (!contentField.empty()) {
const QString content = QString::fromStdWString(contentField);
// Fix: keep a dedicated plain-text snippet for CLI verbose output
// instead of reusing HTML-oriented highlightedContent.
const ContentHighlighter::PlainSnippetResult plainSnippet = ContentHighlighter::plainSnippet(
m_keywords, content, previewLen, previewLen);
const QString highlightedContent = ContentHighlighter::customHighlight(m_keywords, content, previewLen, enableHTML);
resultApi.setHighlightedContent(highlightedContent);
result.setCustomAttribute("plainContentMatch", plainSnippet.content);
result.setCustomAttribute("snippetOffset", plainSnippet.snippetOffset);
}
} catch (const Lucene::LuceneException &e) {
qWarning() << "Exception retrieving content field:" << QString::fromStdWString(e.getError());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -404,19 +404,27 @@ void OcrTextIndexedStrategy::processSearchResults(const Lucene::IndexSearcherPtr

// 设置 OCR 内容结果
OcrTextResultAPI resultApi(result);
result.setCustomAttribute("plainContentMatch", QString());
result.setCustomAttribute("snippetOffset", -1);

// 使用ContentHighlighter命名空间进行高亮
if (enableRetrieval) {
try {
Lucene::String ocrContentField = doc->get(LuceneFieldNames::OcrText::kOcrContents);
if (!ocrContentField.empty()) {
const QString content = QString::fromStdWString(ocrContentField);
// Fix: keep a dedicated plain-text snippet for CLI verbose output
// instead of reusing HTML-oriented highlightedContent.
const ContentHighlighter::PlainSnippetResult plainSnippet = ContentHighlighter::plainSnippet(
m_keywords, content, previewLen, previewLen);
// 设置原始 OCR 内容
resultApi.setOcrContent(content);
// 设置高亮内容
const QString highlightedContent = ContentHighlighter::customHighlight(
m_keywords, content, previewLen, enableHTML);
resultApi.setHighlightedContent(highlightedContent);
result.setCustomAttribute("plainContentMatch", plainSnippet.content);
result.setCustomAttribute("snippetOffset", plainSnippet.snippetOffset);
}
} catch (const Lucene::LuceneException &e) {
qWarning() << "Exception retrieving OCR content field:" << QString::fromStdWString(e.getError());
Expand Down
54 changes: 54 additions & 0 deletions src/dfm-search/dfm-search-lib/utils/contenthighlighter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,15 @@ struct KeywordMatch
QString keyword;
};

int positioningBeforeLength(int maxLength, int positioningMaxLength)
{
const int effectivePositioningLength = qMax(30, positioningMaxLength);
if (maxLength > 0) {
return qMin(effectivePositioningLength / 2, maxLength / 2);
}
return effectivePositioningLength / 2;
}

/**
* @brief 检查给定位置是否是段落的开头
* @param content 原始文本内容
Expand Down Expand Up @@ -153,6 +162,24 @@ KeywordMatch findFirstKeywordMatch(const QString &content, const QStringList &ke
}
return match;
}

KeywordMatch findEarliestKeywordMatch(const QString &content, const QStringList &keywords)
{
KeywordMatch match = { -1, 0, QString() };
for (const QString &keyword : keywords) {
if (keyword.isEmpty()) {
continue;
}

const int pos = content.indexOf(keyword, 0, Qt::CaseInsensitive);
if (pos != -1 && (match.position == -1 || pos < match.position)) {
match.position = pos;
match.length = keyword.length();
match.keyword = keyword;
}
}
return match;
}
} // namespace

QString customHighlight(const QStringList &keywords, const QString &content, int maxLength, bool enableHtml, int positioningMaxLength)
Expand Down Expand Up @@ -242,6 +269,33 @@ QString customHighlight(const QStringList &keywords, const QString &content, int
return resultSnippet;
}

PlainSnippetResult plainSnippet(const QStringList &keywords, const QString &content,
int maxLength, int positioningMaxLength)
{
PlainSnippetResult result;
if (content.isEmpty() || keywords.isEmpty()) {
return result;
}

const KeywordMatch match = findEarliestKeywordMatch(content, keywords);
if (match.position == -1) {
return result;
}

int snippetStart = qMax(0, match.position - positioningBeforeLength(maxLength, positioningMaxLength));
if (maxLength > 0 && content.length() - snippetStart < maxLength) {
snippetStart = qMax(0, content.length() - maxLength);
}

const int snippetLength = maxLength > 0
? qMin(maxLength, content.length() - snippetStart)
: content.length() - snippetStart;

result.content = content.mid(snippetStart, snippetLength);
result.snippetOffset = snippetStart;
return result;
}

QString previewSnippet(const QString &content, int offset, int maxLength,
const QString &keyword, int *keywordOffset)
{
Expand Down
23 changes: 23 additions & 0 deletions src/dfm-search/dfm-search-lib/utils/contenthighlighter.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,12 @@ DFM_SEARCH_BEGIN_NS

namespace ContentHighlighter {

struct PlainSnippetResult
{
QString content;
int snippetOffset = -1;
};

/**
* @brief Extracts and highlights the most relevant text snippet from content based on the provided keywords.
*
Expand All @@ -40,6 +46,23 @@ namespace ContentHighlighter {
*/
QString customHighlight(const QStringList &keywords, const QString &content, int maxLength, bool enableHtml, int positioningMaxLength = 30);

/**
* @brief Extracts a plain-text snippet for CLI verbose output.
*
* Unlike customHighlight(), this helper preserves the original content bytes
* in the returned snippet: it does not simplify whitespace, insert HTML tags,
* or prepend ellipsis. The snippet start is returned so callers can expose the
* offset in the original full text.
*
* @param keywords Search keywords used to anchor the snippet.
* @param content Original document content.
* @param maxLength Maximum length of the returned snippet. <= 0 means until EOF.
* @param positioningMaxLength Window used to position the first content match.
* @return PlainSnippetResult with snippet text and snippetOffset, or empty/-1 if no content match exists.
*/
PlainSnippetResult plainSnippet(const QStringList &keywords, const QString &content,
int maxLength, int positioningMaxLength = 30);

/**
* @brief 精确预览:从 offset 位置截取内容,或从 offset 搜索 keyword 后从匹配位置截取
*
Expand Down
Loading