feat: update web search code

Signed-off-by: yuluo-yx <yuluo08290126@gmail.com>

Author: yuluo-yx

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/service/SAAWebSearchService.java

@@ -81,7 +81,8 @@ public Flux<String> chat(String prompt) {
 		return chatClient.prompt()
 				.advisors(
 						createRetrievalAugmentationAdvisor(),
-						reasoningContentAdvisor,
+						// 不整合到 reasoning content 输出中
+						// reasoningContentAdvisor,
 						simpleLoggerAdvisor
 				).user(prompt)
 				.stream()

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/config/WeSearchConfiguration.java

@@ -1,13 +1,13 @@
 package com.alibaba.cloud.ai.application.websearch.config;
 
 import com.alibaba.cloud.ai.application.websearch.rag.postretrieval.DashScopeDocumentRanker;
+import com.alibaba.cloud.ai.application.websearch.rag.preretrieval.query.expansion.MultiQueryExpander;
 import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions;
 import com.alibaba.cloud.ai.model.RerankModel;
 
 import org.springframework.ai.chat.client.ChatClient;
 import org.springframework.ai.chat.client.advisor.SimpleLoggerAdvisor;
 import org.springframework.ai.chat.prompt.PromptTemplate;
-import org.springframework.ai.rag.preretrieval.query.expansion.MultiQueryExpander;
 import org.springframework.ai.rag.preretrieval.query.expansion.QueryExpander;
 import org.springframework.ai.rag.preretrieval.query.transformation.QueryTransformer;
 import org.springframework.ai.rag.preretrieval.query.transformation.RewriteQueryTransformer;

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/data/DataClean.java

@@ -29,19 +29,27 @@ public List<Document> getData(GenericSearchResult respData) {
 
 		List<Document> documents = new ArrayList<>();
 
-		// 1. 获取 QueryContext 的 metadata
 		Map<String, Object> metadata = getQueryMetadata(respData);
 
 		for (ScorePageItem pageItem : respData.getPageItems()) {
 
-			// 获取每个 pages 的 metadata
 			Map<String, Object> pageItemMetadata = getPageItemMetadata(pageItem);
-			// 获取 text
-			String text = getText(pageItem);
-			// 获取 media Document 限制，media 和 text 只能有一个
-			// Media media = getMedia(pageItem);
-			// 获取浏览器 score
 			Double score = getScore(pageItem);
+			String text = getText(pageItem);
+
+			if (Objects.equals("", text)) {
+
+				Media media = getMedia(pageItem);
+				Document document = new Document.Builder()
+						.metadata(metadata)
+						.metadata(pageItemMetadata)
+						.media(media)
+						.score(score)
+						.build();
+
+				documents.add(document);
+				break;
+			}
 
 			Document document = new Document.Builder()
 					.metadata(metadata)
@@ -126,7 +134,18 @@ private Media getMedia(ScorePageItem pageItem) {
 
 	private String getText(ScorePageItem pageItem) {
 
-		return pageItem.getMainText();
+		if (Objects.nonNull(pageItem.getMainText())) {
+
+			String mainText = pageItem.getMainText();
+
+			mainText = mainText.replaceAll("<[^>]+>", "");
+			mainText = mainText.replaceAll("[\\n\\t\\r]+", " ");
+			mainText = mainText.replaceAll("[\\u200B-\\u200D\\uFEFF]", "");
+
+			return mainText.trim();
+		}
+
+		return "";
 	}
 
 	public List<Document> limitResults(List<Document> documents, int minResults) {

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/rag/WebSearchRetriever.java

@@ -42,7 +42,6 @@ public List<Document> retrieve(
 
 		// 搜索
 		GenericSearchResult searchResp = searchEngine.search(query.text());
-		logger.debug("search response: {}", searchResp);
 
 		// 清洗数据
 		List<Document> cleanerData = dataCleaner.getData(searchResp);

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/rag/join/ConcatenationDocumentJoiner.java

@@ -10,12 +10,14 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.springframework.ai.document.Document;
 import org.springframework.ai.rag.Query;
 import org.springframework.ai.rag.retrieval.join.DocumentJoiner;
+import org.springframework.lang.Nullable;
 import org.springframework.util.Assert;
 
 /**
@@ -27,8 +29,11 @@ public class ConcatenationDocumentJoiner implements DocumentJoiner {
 
 	private static final Logger logger = LoggerFactory.getLogger(ConcatenationDocumentJoiner.class);
 
+	@NotNull
 	@Override
-	public List<Document> join(Map<Query, List<List<Document>>> documentsForQuery) {
+	public List<Document> join(
+			@Nullable Map<Query, List<List<Document>>> documentsForQuery
+	) {
 
 		Assert.notNull(documentsForQuery, "documentsForQuery cannot be null");
 		Assert.noNullElements(documentsForQuery.keySet(), "documentsForQuery cannot contain null keys");

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/rag/postretrieval/DashScopeDocumentRanker.java

@@ -10,10 +10,15 @@
 import com.alibaba.cloud.ai.model.RerankModel;
 import com.alibaba.cloud.ai.model.RerankRequest;
 import com.alibaba.cloud.ai.model.RerankResponse;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.springframework.ai.document.Document;
 import org.springframework.ai.rag.Query;
 import org.springframework.ai.rag.postretrieval.ranking.DocumentRanker;
+import org.springframework.lang.Nullable;
+import org.springframework.util.StringUtils;
 
 /**
  * @author yuluo
@@ -22,14 +27,24 @@
 
 public class DashScopeDocumentRanker implements DocumentRanker {
 
+	private static final Logger logger = LoggerFactory.getLogger(DashScopeDocumentRanker.class);
+
 	private final RerankModel rerankModel;
 
 	public DashScopeDocumentRanker(RerankModel rerankModel) {
 		this.rerankModel = rerankModel;
 	}
 
+	@NotNull
 	@Override
-	public List<Document> rank(Query query, List<Document> documents) {
+	public List<Document> rank(
+			@Nullable Query query,
+			@Nullable List<Document> documents
+	) {
+
+		if (documents == null || documents.isEmpty()) {
+			return new ArrayList<>();
+		}
 
 		try {
 			List<Document> reorderDocs = new ArrayList<>();
@@ -39,27 +54,35 @@ public List<Document> rank(Query query, List<Document> documents) {
 					.withTopN(documents.size())
 					.build();
 
-			// 组装参数调用 rerankModel
-			RerankRequest rerankRequest = new RerankRequest(
-					query.text(),
-					documents,
-					rerankOptions
-			);
-			RerankResponse rerankResp = rerankModel.call(rerankRequest);
+			if (Objects.nonNull(query) && StringUtils.hasText(query.text())) {
+				// 组装参数调用 rerankModel
+				RerankRequest rerankRequest = new RerankRequest(
+						query.text(),
+						documents,
+						rerankOptions
+				);
+				RerankResponse rerankResp = rerankModel.call(rerankRequest);
 
-			rerankResp.getResults().forEach(res -> {
-				Document outputDocs = res.getOutput();
+				rerankResp.getResults().forEach(res -> {
+					Document outputDocs = res.getOutput();
 
-				// 查找并添加到新的 list 中
-				Optional<Document> foundDocsOptional = documents.stream()
-						.filter(doc -> Objects.equals(doc.getId(), outputDocs.getId()))
-						.findFirst();
+					// 查找并添加到新的 list 中
+					Optional<Document> foundDocsOptional = documents.stream()
+							.filter(doc ->
+							{
+								// debug rerank output.
+								logger.debug("doc id: {}, outputDocs id: {}", doc.getId(), outputDocs.getId());
+								return Objects.equals(doc.getId(), outputDocs.getId());
+							})
+							.findFirst();
 
-				foundDocsOptional.ifPresent(reorderDocs::add);
-			});
+					foundDocsOptional.ifPresent(reorderDocs::add);
+				});
+			}
 
 			return reorderDocs;
-		} catch (Exception e) {
+		}
+		catch (Exception e) {
 			// 根据异常类型做进一步处理
 			throw new SAAAppException(e.getMessage());
 		}

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/rag/preretrieval/query/expansion/MultiQueryExpander.java

@@ -5,6 +5,7 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,7 +46,7 @@ public class MultiQueryExpander implements QueryExpander {
 
 	private static final Boolean DEFAULT_INCLUDE_ORIGINAL = true;
 
-	private static final Integer DEFAULT_NUMBER_OF_QUERIES = 5;
+	private static final Integer DEFAULT_NUMBER_OF_QUERIES = 3;
 
 	private final ChatClient chatClient;
 
@@ -61,6 +62,7 @@ public MultiQueryExpander(
 			@Nullable Boolean includeOriginal,
 			@Nullable Integer numberOfQueries
 	) {
+
 		Assert.notNull(chatClientBuilder, "ChatClient.Builder must not be null");
 
 		this.chatClient = chatClientBuilder.build();
@@ -71,8 +73,9 @@ public MultiQueryExpander(
 		PromptAssert.templateHasRequiredPlaceholders(this.promptTemplate, "number", "query");
 	}
 
+	@NotNull
 	@Override
-	public List<Query> expand(Query query) {
+	public List<Query> expand(@Nullable Query query) {
 
 		Assert.notNull(query, "Query must not be null");
 
@@ -95,7 +98,9 @@ public List<Query> expand(Query query) {
 
 		if (CollectionUtils.isEmpty(queryVariants) || this.numberOfQueries != queryVariants.size()) {
 
-			logger.warn("Query expansion result dose not contain the requested {} variants for query: {}. is return.", this.numberOfQueries, query.text());
+			logger.warn("Query expansion result dose not contain the requested {} variants for query: {}. is return.",
+					this.numberOfQueries, query.text());
+
 			return List.of(query);
 		}
 
@@ -109,10 +114,16 @@ public List<Query> expand(Query query) {
 			logger.debug("Including original query in the expanded queries for query: {}", query.text());
 			queries.add(0, query);
 		}
+		
+		logger.debug("Rewrite queries: {}", queries);
 
 		return queries;
 	}
 
+	public static Builder builder() {
+		return new Builder();
+	}
+
 	public static final class Builder {
 
 		private ChatClient.Builder chatClientBuilder;

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/websearch/rag/prompt/CustomContextQueryAugmenter.java

@@ -5,6 +5,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -71,18 +72,25 @@ public CustomContextQueryAugmenter(
 		PromptAssert.templateHasRequiredPlaceholders(this.promptTemplate, "query", "context");
 	}
 
+	@NotNull
 	@Override
-	public Query augment(Query query, List<Document> documents) {
+	public Query augment(
+			@Nullable Query query,
+			@Nullable List<Document> documents
+	) {
 
 		Assert.notNull(query, "Query must not be null");
 		Assert.notNull(documents, "Documents must not be null");
 
 		logger.debug("Augmenting query: {}", query);
 
 		if (documents.isEmpty()) {
+			logger.debug("No documents found. Augmenting query with empty context.");
 			return augmentQueryWhenEmptyContext(query);
 		}
 
+		logger.debug("Documents found. Augmenting query with context.");
+
 		// 1. collect content from documents.
 		AtomicInteger idCounter = new AtomicInteger(1);
 		String documentContext = documents.stream()

spring-ai-alibaba-integration-example/backend/src/main/resources/application-dev.yml

@@ -42,6 +42,6 @@ spring:
 # enabled debug log out.
 logging:
   level:
-    org.springframework.ai: debug
-    com.alibaba.dashscope.api: debug
+#    org.springframework.ai: debug
+#    com.alibaba.dashscope.api: debug
     com.alibaba.cloud.ai.application.websearch: debug

spring-ai-alibaba-integration-example/backend/src/main/resources/web-search.http

@@ -0,0 +1 @@
+http://localhost:8080/api/v1/search?query="杭州有什么推荐旅游的地方吗？"