Skip to content

Commit 074fb46

Browse files
LambduaLambdua
authored
Merge pull request #78 from Lambdua/#77_userMessageBuilder
#77 add UserMessageBuilder
2 parents a5c9b0c + 4d9b95a commit 074fb46

File tree

5 files changed

+291
-8
lines changed

5 files changed

+291
-8
lines changed

‎api/src/main/java/com/theokanning/openai/completion/chat/ImageContent.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
/**
1717
* @author LiangTao
1818
* @date 2024年04月10 10:26
19+
* @deprecated use {@link MultiMediaContent},the name is not accurate,use new class {@link MultiMediaContent} instead and the refactoring is done
1920
**/
2021
@Data
2122
@AllArgsConstructor
2223
@NoArgsConstructor
24+
@Deprecated
2325
public class ImageContent {
2426

2527
/**

‎api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,4 @@ public class ImageUrl {
2121
public ImageUrl(String url) {
2222
this.url = url;
2323
}
24-
25-
2624
}

‎api/src/main/java/com/theokanning/openai/completion/chat/MultiMediaContent.java

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package com.theokanning.openai.completion.chat;
2+
3+
import com.fasterxml.jackson.annotation.JsonInclude;
4+
import com.fasterxml.jackson.annotation.JsonProperty;
5+
import com.theokanning.openai.assistants.message.content.ImageFile;
6+
import lombok.AllArgsConstructor;
7+
import lombok.Data;
8+
import lombok.NoArgsConstructor;
9+
import lombok.NonNull;
10+
11+
import java.io.IOException;
12+
import java.nio.file.Files;
13+
import java.nio.file.Path;
14+
import java.util.Base64;
15+
16+
/**
17+
* @author LiangTao
18+
* @date 2024年04月10 10:26
19+
**/
20+
@Data
21+
@AllArgsConstructor
22+
@NoArgsConstructor
23+
public class MultiMediaContent {
24+
25+
/**
26+
* The type of the content. Either "text", "image_url" or "input_audio".
27+
*/
28+
@NonNull
29+
private String type;
30+
31+
@JsonInclude(JsonInclude.Include.NON_NULL)
32+
private String text;
33+
34+
@JsonInclude(JsonInclude.Include.NON_NULL)
35+
@JsonProperty("image_url")
36+
private ImageUrl imageUrl;
37+
38+
@JsonInclude(JsonInclude.Include.NON_NULL)
39+
@JsonProperty("image_file")
40+
private ImageFile imageFile;
41+
42+
@JsonInclude(JsonInclude.Include.NON_NULL)
43+
@JsonProperty("input_audio")
44+
private InputAudio inputAudio;
45+
46+
public MultiMediaContent(String text) {
47+
this.type = "text";
48+
this.text = text;
49+
}
50+
51+
public MultiMediaContent(ImageUrl imageUrl) {
52+
this.type = "image_url";
53+
this.imageUrl = imageUrl;
54+
}
55+
56+
public MultiMediaContent(InputAudio inputAudio) {
57+
this.type = "input_audio";
58+
this.inputAudio = inputAudio;
59+
}
60+
61+
/**
62+
* build an image content from a file path, detail is auto
63+
* @author liangtao
64+
* @date 2024/11/28
65+
* @param imagePath
66+
* @return com.theokanning.openai.completion.chat.ImageContent
67+
**/
68+
public static MultiMediaContent ofImagePath(Path imagePath){
69+
return ofImagePath(imagePath,"auto");
70+
}
71+
72+
/**
73+
* build an image content from a file path,Specify detail
74+
* @author liangtao
75+
* @date 2024/11/28
76+
* @param imagePath
77+
* @param detail level: "auto", "low", "high"
78+
* @return com.theokanning.openai.completion.chat.ImageContent
79+
**/
80+
public static MultiMediaContent ofImagePath(Path imagePath, String detail){
81+
String imagePathString = imagePath.toAbsolutePath().toString();
82+
String extension = imagePathString.substring(imagePathString.lastIndexOf('.') + 1);
83+
ImageUrl imageUrl = new ImageUrl("data:image/" + extension + ";base64," + encode2base64(imagePath),detail);
84+
return new MultiMediaContent(imageUrl);
85+
}
86+
87+
/**
88+
* build an image content from a url
89+
* @param imageUrl url
90+
* @param detail level: "auto", "low", "high"
91+
*/
92+
public static MultiMediaContent ofImageUrl(String imageUrl, String detail) {
93+
return new MultiMediaContent(new ImageUrl(imageUrl, detail));
94+
}
95+
96+
public static MultiMediaContent ofImageUrl(String imageUrl) {
97+
return ofImageUrl(imageUrl, "auto");
98+
}
99+
100+
101+
102+
public static MultiMediaContent ofAudioPath(Path inputAudioPath) {
103+
String inputAudioPathString = inputAudioPath.toAbsolutePath().toString();
104+
String extension = inputAudioPathString.substring(inputAudioPathString.lastIndexOf('.') + 1);
105+
String base64 = encode2base64(inputAudioPath);
106+
InputAudio inputAudio = new InputAudio(base64, extension);
107+
return new MultiMediaContent(inputAudio);
108+
}
109+
110+
111+
private static String encode2base64(Path path) {
112+
byte[] fileContent;
113+
try {
114+
fileContent = Files.readAllBytes(path);
115+
return Base64.getEncoder().encodeToString(fileContent);
116+
} catch (IOException e) {
117+
throw new RuntimeException(e);
118+
}
119+
}
120+
}

‎api/src/main/java/com/theokanning/openai/completion/chat/UserMessage.java

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@
77
import lombok.Data;
88
import lombok.NoArgsConstructor;
99

10-
import java.io.IOException;
11-
import java.nio.file.Files;
1210
import java.nio.file.Path;
13-
import java.util.*;
11+
import java.util.Arrays;
12+
import java.util.Collection;
13+
import java.util.List;
14+
import java.util.Optional;
1415
import java.util.stream.Collectors;
1516

1617
/**
@@ -34,13 +35,21 @@ public UserMessage(String content) {
3435
this.content = content;
3536
}
3637

38+
public static UserMessageBuilder builder() {
39+
return new UserMessageBuilder();
40+
}
41+
42+
/**
43+
*
44+
* @param imageContents
45+
*/
46+
@Deprecated
3747
public UserMessage(List<ImageContent> imageContents) {
3848
this.content = imageContents;
3949
}
4050

4151

4252

43-
4453
@Override
4554
@JsonIgnore
4655
public String getTextContent() {
@@ -61,45 +70,56 @@ public String getTextContent() {
6170

6271
/**
6372
* 构件一个图片识别请求消息,支持多个图片
64-
*
73+
* @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead
6574
* @param prompt query text
6675
* @param imageUrls image urls
6776
* @return com.theokanning.openai.completion.chat.UserMessage
6877
* @author liangtao
6978
* @date 2024/4/12
7079
**/
80+
@Deprecated
7181
public static UserMessage buildImageMessage(String prompt, String... imageUrls) {
7282
List<ImageContent> imageContents = Arrays.stream(imageUrls).map(url -> new ImageContent(new ImageUrl(url))).collect(Collectors.toList());
7383
imageContents.add(0, new ImageContent(prompt));
7484
return new UserMessage(imageContents);
7585
}
7686

7787
/**
78-
* 构件一个图片识别请求消息,支持多个图片
88+
* 构件一个图片识别请求消息,支持多个图片,detail默认为auto
89+
* @deprecated use {@link UserMessageBuilder#buildImageMessage(String, String...)} instead
7990
* @author liangtao
8091
* @date 2024/8/15
8192
* @param prompt query text
8293
* @param imagePaths 文件本地路径
8394
* @return com.theokanning.openai.completion.chat.UserMessage
8495
**/
96+
@Deprecated
8597
public static UserMessage buildImageMessage(String prompt, Path... imagePaths) {
8698
List<ImageContent> imageContents = Arrays.stream(imagePaths).map(ImageContent::ofImagePath).collect(Collectors.toList());
8799
imageContents.add(0, new ImageContent(prompt));
88100
return new UserMessage(imageContents);
89101
}
90102

103+
104+
91105
/**
92106
* 构建一个音频识别请求消息,支持多个音频
107+
* @deprecated use {@link UserMessageBuilder#buildAudioMessage(String, Path...)} instead
93108
* @param prompt query text
94109
* @param inputAudioPaths 音频文件本地路径
95110
* @return com.theokanning.openai.completion.chat.UserMessage
96111
* @author Allen Hu
97112
* @date 2024/11/6
98113
*/
114+
@Deprecated
99115
public static UserMessage buildInputAudioMessage(String prompt, Path... inputAudioPaths) {
100116
List<ImageContent> imageContents = Arrays.stream(inputAudioPaths).map(ImageContent::ofAudioPath).collect(Collectors.toList());
101117
imageContents.add(0, new ImageContent(prompt));
102118
return new UserMessage(imageContents);
103119
}
120+
121+
public static void main(String[] args) {
122+
// UserMessage.builder().withMultiMediaContent()
123+
}
104124
}
105125

‎api/src/main/java/com/theokanning/openai/completion/chat/UserMessageBuilder.java

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package com.theokanning.openai.completion.chat;
2+
3+
import lombok.extern.slf4j.Slf4j;
4+
5+
import java.nio.file.Path;
6+
import java.util.ArrayList;
7+
import java.util.Arrays;
8+
import java.util.List;
9+
import java.util.stream.Collectors;
10+
11+
/**
12+
* @author LiangTao
13+
* @date 2024年11月29 15:18
14+
**/
15+
@Slf4j
16+
public class UserMessageBuilder {
17+
18+
private final UserMessage userMessage;
19+
20+
public UserMessageBuilder() {
21+
userMessage = new UserMessage();
22+
}
23+
24+
public UserMessageBuilder withName(String name) {
25+
userMessage.setName(name);
26+
return this;
27+
}
28+
29+
/**
30+
* Set the text content of the message.
31+
* @param text the text content of the message
32+
*/
33+
public UserMessageBuilder withTextMessage(String text) {
34+
userMessage.setContent(text);
35+
return this;
36+
}
37+
38+
39+
/**
40+
* build the message with image/audio content
41+
* @param multiMediaContent the image content or audio content of the message
42+
* @return the UserMessageBuilder
43+
*/
44+
public UserMessageBuilder withMultiMediaContent(List<MultiMediaContent> multiMediaContent) {
45+
userMessage.setContent(multiMediaContent);
46+
return this;
47+
}
48+
49+
50+
/**
51+
* This method supports adding image/audio message content to assist the component UserMessage
52+
* @param multiMediaContents image audio message content
53+
*/
54+
public UserMessageBuilder addMultiMediaContents(MultiMediaContent... multiMediaContents) {
55+
Object content = userMessage.getContent();
56+
if (content==null){
57+
ArrayList<MultiMediaContent> multiMediaContentList = new ArrayList<>();
58+
userMessage.setContent(multiMediaContentList);
59+
}else {
60+
if (content instanceof String){
61+
log.error("The content of the message is text, can not add image content");
62+
return this;
63+
}
64+
}
65+
List<MultiMediaContent> multiMediaContentList = (List<MultiMediaContent>) content;
66+
multiMediaContentList.addAll(Arrays.asList(multiMediaContents));
67+
return this;
68+
}
69+
70+
/**
71+
* Used for component image messages, supporting multiple images
72+
* @param prompt prompt
73+
* @param imageUrl image url or base64 image data
74+
*/
75+
public UserMessage buildImageMessage(String prompt, String imageUrl) {
76+
return buildImageMessageWithDetail(prompt,"auto",imageUrl);
77+
}
78+
79+
/**
80+
* Used for component image messages, supporting multiple images
81+
* @param prompt prompt
82+
* @param detail level: "auto", "low", "high"
83+
* @param imageUrls image urls or base64 image data
84+
*/
85+
public UserMessage buildImageMessageWithDetail(String prompt, String detail,String... imageUrls) {
86+
List<MultiMediaContent> imageContents = Arrays.stream(imageUrls).map(url -> MultiMediaContent.ofImageUrl(url,detail)).collect(Collectors.toList());
87+
imageContents.add(0, new MultiMediaContent(prompt));
88+
userMessage.setContent(imageContents);
89+
return build();
90+
}
91+
92+
/**
93+
* Used for component image messages, supporting multiple images
94+
* @param prompt prompt
95+
* @param imageUrls image urls or base64 image data
96+
*/
97+
public UserMessage buildImageMessage(String prompt, String... imageUrls) {
98+
return buildImageMessageWithDetail(prompt,"auto",imageUrls);
99+
}
100+
101+
/**
102+
* Used for component image messages, supporting multiple images
103+
* @param prompt prompt
104+
* @param detail level: "auto", "low", "high"
105+
* @param imagePaths image file paths
106+
*/
107+
public UserMessage buildImageMessageWithDetail(String prompt, String detail, Path... imagePaths) {
108+
List<MultiMediaContent> imageContents = Arrays.stream(imagePaths).map(path -> MultiMediaContent.ofImagePath(path,detail)).collect(Collectors.toList());
109+
imageContents.add(0, new MultiMediaContent(prompt));
110+
userMessage.setContent(imageContents);
111+
return build();
112+
}
113+
114+
115+
/**
116+
* Used for component image messages, supporting multiple images
117+
* @param prompt prompt
118+
* @param imagePaths image file paths
119+
*/
120+
public UserMessage buildImageMessage(String prompt, Path... imagePaths) {
121+
return buildImageMessageWithDetail(prompt,"auto",imagePaths);
122+
}
123+
124+
/**
125+
* Used for component audio messages, supporting multiple audio
126+
* @param prompt prompt
127+
* @param audioPaths audio file paths
128+
*/
129+
public UserMessage buildAudioMessage(String prompt, Path... audioPaths) {
130+
List<MultiMediaContent> audioContents = Arrays.stream(audioPaths).map(MultiMediaContent::ofAudioPath).collect(Collectors.toList());
131+
audioContents.add(0, new MultiMediaContent(prompt));
132+
userMessage.setContent(audioContents);
133+
return build();
134+
}
135+
136+
137+
138+
public UserMessage build() {
139+
return userMessage;
140+
}
141+
142+
143+
}

0 commit comments

Comments
 (0)