Merge pull request #61 from springaialibaba/0228-yuluo/add-more-chatclient

feat: update playground code

Author: yuluo-yx

spring-ai-alibaba-integration-example/backend/request/spring-ai-alibaba-playground.json

@@ -17,19 +17,23 @@
 
 package com.alibaba.cloud.ai.application.config;
 
+import java.io.File;
+
 import com.alibaba.cloud.ai.application.utils.FilesUtils;
 import jakarta.annotation.PreDestroy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
+import org.springframework.stereotype.Component;
 
 /**
  * @author yuluo
  * @author <a href="mailto:yuluo08290126@gmail.com">yuluo</a>
  */
 
+@Component
 public class TmpFolderConfig implements ApplicationRunner {
 
 	private static final String ImageTmpFolder = "tmp/image";
@@ -43,16 +47,20 @@ public void run(ApplicationArguments args) throws Exception {
 
 		logger.info("Init tmp folder");
 
-		FilesUtils.initTmpFolder(ImageTmpFolder);
-		FilesUtils.initTmpFolder(AudioTmpFolder);
+		FilesUtils.initTmpFolder(System.getProperty("user.dir") + "/" + ImageTmpFolder);
+		FilesUtils.initTmpFolder(System.getProperty("user.dir") + "/" +AudioTmpFolder);
 
 		logger.info("Init tmp folder");
 	}
 
 	@PreDestroy
 	public void destroy() {
 
-		// todo: delete tmp folder
+		FilesUtils.deleteDirectory(new File(System.getProperty("user.dir") + "/" +ImageTmpFolder));
+		FilesUtils.deleteDirectory(new File(System.getProperty("user.dir") + "/" +AudioTmpFolder));
+		FilesUtils.deleteDirectory(new File(System.getProperty("user.dir") + "/" +"tmp"));
+
+		logger.info("Delete tmp folder");
 	}
 
 }

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/config/TmpFolderConfig.java

@@ -26,7 +26,6 @@
 import reactor.core.publisher.Flux;
 
 import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -56,35 +55,46 @@ public SAAAudioController(SAAAudioService audioService) {
 	@UserIp
 	@PostMapping("/audio2text")
 	@Operation(summary = "DashScope Audio Transcription")
-	public Flux<Result<String>> audioToText(@RequestParam("audio") MultipartFile audio) {
+	public Flux<Result<String>> audioToText(
+			@RequestParam("audio") MultipartFile audio
+	) {
 
 		if (audio.isEmpty()) {
 			return Flux.just(Result.failed("No audio file provided"));
 		}
 
-		return audioService.audio2text(audio).map(Result::success);
+		Flux<Result<String>> res;
+		try {
+			res = audioService.audio2text(audio).map(Result::success);
+		} catch (Exception e) {
+			return Flux.just(Result.failed("Failed to transcribe audio: " + e.getMessage()));
+		}
+
+		return res;
 	}
 
 	/**
 	 * text2audio
 	 * 用于将文本转换为语音输出
 	 */
 	@UserIp
-	@GetMapping("/text2audio/{prompt}")
+	@GetMapping("/text2audio")
 	@Operation(summary = "DashScope Speech Synthesis")
-	public Result<byte[]> textToAudio(@PathVariable("prompt") String prompt) {
+	public Result<byte[]> textToAudio(
+			@RequestParam("prompt") String prompt
+	) {
 
 		if (!ValidText.isValidate(prompt)) {
 			return Result.failed("No chat prompt provided");
 		}
 
 		byte[] audioData = audioService.text2audio(prompt);
 
-//		测试验证音频数据是否为空
-//		try (FileOutputStream fos = new FileOutputStream("audio.wav")) {
+		// 测试验证音频数据是否为空
+//		try (FileOutputStream fos = new FileOutputStream("tmp/audio/test-audio.wav")) {
 //			fos.write(audioData);
 //		} catch (IOException e) {
-//			return Result.failed("Failed to save audio file: " + e.getMessage());
+//			return Result.failed("Test save audio file: " + e.getMessage());
 //		}
 
 		return Result.success(audioData);

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/controller/SAAAudioController.java

@@ -58,7 +58,7 @@ public SAAImageController(SAAImageService imageService) {
 	@PostMapping("/image2text")
 	@Operation(summary = "DashScope Image Recognition")
 	public Flux<Result<String>> image2text(
-			@RequestParam("prompt") String prompt,
+			@RequestParam(value = "prompt", required = false) String prompt,
 			@RequestParam("image") MultipartFile image
 	) {
 
@@ -70,11 +70,18 @@ public Flux<Result<String>> image2text(
 			prompt = "请你用一句话描述这张图片";
 		}
 
-		return imageService.image2Text(prompt, image).map(Result::success);
+		Flux<Result<String>> res;
+		try {
+			 res = imageService.image2Text(prompt, image).map(Result::success);
+		} catch (Exception e) {
+			return Flux.just(Result.failed(e.getMessage()));
+		}
+
+		return res;
 	}
 
 	@UserIp
-	@GetMapping("/text2Image")
+	@GetMapping("/text2image")
 	@Operation(summary = "DashScope Image Generation")
 	public Result<Void> text2Image(
 			@RequestParam("prompt") String prompt,

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/controller/SAAImageController.java

@@ -25,6 +25,7 @@
 
 import com.alibaba.cloud.ai.application.exception.SAAAIException;
 import com.alibaba.cloud.ai.application.exception.SAAAppException;
+import com.alibaba.cloud.ai.application.utils.FilesUtils;
 import com.alibaba.cloud.ai.dashscope.audio.DashScopeAudioTranscriptionOptions;
 import com.alibaba.cloud.ai.dashscope.audio.synthesis.SpeechSynthesisModel;
 import com.alibaba.cloud.ai.dashscope.audio.synthesis.SpeechSynthesisPrompt;
@@ -50,6 +51,8 @@ public class SAAAudioService {
 
 	private final SpeechSynthesisModel speechSynthesisModel;
 
+	private final String DEFAULT_MODEL = "paraformer-realtime-v2";
+
 	public SAAAudioService(AudioTranscriptionModel transcriptionModel, SpeechSynthesisModel speechSynthesisModel) {
 
 		this.transcriptionModel = transcriptionModel;
@@ -97,25 +100,19 @@ public byte[] text2audio(String text) {
 	/**
 	 * 将语音转为文本
 	 */
-	public Flux<String> audio2text(MultipartFile audio) {
+	public Flux<String> audio2text(MultipartFile file) throws IOException {
 
 		CountDownLatch latch = new CountDownLatch(1);
 		StringBuilder stringBuilder = new StringBuilder();
 
-		File tempFile;
-		try {
-			tempFile = File.createTempFile("audio", ".pcm");
-			audio.transferTo(tempFile);
-		}
-		catch (IOException e) {
-			throw new SAAAppException("Failed to create temporary file " + e.getMessage());
-		}
+		String filePath = System.getProperty("user.dir") + "/" + "tmp/audio/" + file.getOriginalFilename();
+		FilesUtils.saveTempImage(file, filePath);
 
 		Flux<AudioTranscriptionResponse> response = transcriptionModel.stream(
 				new AudioTranscriptionPrompt(
-						new FileSystemResource(tempFile),
+						new FileSystemResource(filePath),
 						DashScopeAudioTranscriptionOptions.builder()
-								.withModel("paraformer-realtime-v2")
+								.withModel(DEFAULT_MODEL)
 								.withSampleRate(16000)
 								.withFormat(DashScopeAudioTranscriptionOptions.AudioFormat.PCM)
 								.withDisfluencyRemovalEnabled(false)
@@ -133,7 +130,7 @@ public Flux<String> audio2text(MultipartFile audio) {
 			throw new SAAAIException("Transcription was interrupted " + e.getMessage());
 		}
 		finally {
-			tempFile.delete();
+			new File(filePath).delete();
 		}
 
 		return Flux.just(stringBuilder.toString());

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/service/SAAAudioService.java

@@ -22,9 +22,10 @@
 import java.net.URI;
 import java.net.URL;
 import java.util.List;
-import java.util.Objects;
 
+import com.alibaba.cloud.ai.application.utils.FilesUtils;
 import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel;
+import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions;
 import com.alibaba.cloud.ai.dashscope.chat.MessageFormat;
 import jakarta.servlet.http.HttpServletResponse;
 import reactor.core.publisher.Flux;
@@ -52,32 +53,54 @@
 @Service
 public class SAAImageService {
 
+	private static final String DEFAULT_MODEL = "qwen-vl-max-latest";
+
+	/**
+	 * 图片生成文字
+	 */
 	private final ImageModel imageModel;
 
+	/**
+	 * 多模态支持，用来解析图片
+	 */
 	private final ChatClient daschScopeChatClient;
 
-	public SAAImageService(ImageModel imageModel, ChatModel chatModel) {
+	public SAAImageService(
+			ImageModel imageModel,
+			ChatModel chatModel
+	) {
 
 		this.imageModel = imageModel;
 		this.daschScopeChatClient = ChatClient
 				.builder(chatModel)
 				.build();
 	}
 
-	public Flux<String> image2Text(String prompt, MultipartFile file) {
+	public Flux<String> image2Text(String prompt, MultipartFile file) throws IOException {
+
+		String filePath = System.getProperty("user.dir") + "/" + "tmp/image/" + file.getOriginalFilename();
+		FilesUtils.saveTempImage(file, filePath);
 
 		UserMessage message = new UserMessage(
 				prompt,
 				new Media(
 						MimeTypeUtils.IMAGE_PNG,
-						new FileSystemResource(Objects.requireNonNull(file.getOriginalFilename()))
+						new FileSystemResource(filePath)
 				)
 		);
 		message.getMetadata().put(DashScopeChatModel.MESSAGE_FORMAT, MessageFormat.IMAGE);
 
 		List<ChatResponse> response = daschScopeChatClient.prompt(
-				new Prompt(message)
-		).stream().chatResponse().collectList().block();
+						new Prompt(
+								message,
+								DashScopeChatOptions.builder()
+										.withModel(DEFAULT_MODEL)
+										.withMultiModel(true)
+										.build())
+				).stream()
+				.chatResponse()
+				.collectList()
+				.block();
 
 		StringBuilder result = new StringBuilder();
 		if (response != null) {

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/service/SAAImageService.java

@@ -18,8 +18,13 @@
 package com.alibaba.cloud.ai.application.utils;
 
 import java.io.File;
+import java.io.IOException;
 
 import com.alibaba.cloud.ai.application.exception.SAAAppException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.springframework.web.multipart.MultipartFile;
 
 /**
  * @author yuluo
@@ -28,6 +33,8 @@
 
 public final class FilesUtils {
 
+	private static final Logger logger = LoggerFactory.getLogger(FilesUtils.class);
+
 	private FilesUtils() {
 	}
 
@@ -41,10 +48,38 @@ public static void initTmpFolder(String path) {
 			throw new SAAAppException("path is null or empty");
 		}
 
-		File imageTmpFolder = new File(path);
-		if (!imageTmpFolder.exists()) {
-			imageTmpFolder.mkdirs();
+		File tmpFolder = new File(path);
+		if (!tmpFolder.exists()) {
+			tmpFolder.mkdirs();
 		}
+
+		logger.info("Init tmp folder: {}", tmpFolder.getAbsolutePath());
+	}
+
+	public static void deleteDirectory(File directory) {
+
+		if (directory.exists()) {
+			File[] files = directory.listFiles();
+			if (files != null) {
+				for (File file : files) {
+					if (file.isDirectory()) {
+						deleteDirectory(file);
+					} else {
+						file.delete();
+					}
+				}
+			}
+
+			directory.delete();
+		}
+	}
+
+	/**
+	 * save file to tmp folder
+	 */
+	public static void saveTempImage(MultipartFile file, String path) throws IOException {
+
+		file.transferTo(new File(path));
 	}
 
 }

spring-ai-alibaba-integration-example/backend/src/main/java/com/alibaba/cloud/ai/application/utils/FilesUtils.java

@@ -41,6 +41,7 @@
 		<module>rag-example</module>
 		<module>rag-milvus-example</module>
 		<module>rag-pgvector-example</module>
+		<module>web-search</module>
 	</modules>
 
 	<build>

spring-ai-alibaba-rag-example/pom.xml

@@ -0,0 +1,3 @@
+# Spring AI Alibaba Web Search
+
+building......