Skip to content

Commit dc50053

Browse files
TutorialworkTutorialwork
committed
Get rid of random user agent library and implement own solution
1 parent d9226e1 commit dc50053

File tree

7 files changed

+165
-10
lines changed

7 files changed

+165
-10
lines changed

‎pom.xml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,16 @@
4444
<artifactId>okhttp</artifactId>
4545
<version>4.12.0</version>
4646
</dependency>
47-
<dependency>
48-
<groupId>com.sezinkarli</groupId>
49-
<artifactId>random-user-agent-generator</artifactId>
50-
<version>1.3</version>
51-
</dependency>
5247
<dependency>
5348
<groupId>dev.failsafe</groupId>
5449
<artifactId>failsafe</artifactId>
5550
<version>3.3.2</version>
5651
</dependency>
52+
<dependency>
53+
<groupId>com.google.code.gson</groupId>
54+
<artifactId>gson</artifactId>
55+
<version>2.12.1</version>
56+
</dependency>
5757

5858
<dependency>
5959
<groupId>org.projectlombok</groupId>

‎src/main/java/dev/manuelschuler/kleinanzeigenadsrenewer/config/UserAgentConfig.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
package dev.manuelschuler.kleinanzeigenadsrenewer.config;
2+
3+
import com.google.gson.Gson;
4+
import dev.manuelschuler.kleinanzeigenadsrenewer.model.UserAgent;
5+
import lombok.AllArgsConstructor;
6+
import lombok.Data;
7+
import org.springframework.context.annotation.Bean;
8+
import org.springframework.context.annotation.Configuration;
9+
import org.springframework.core.io.Resource;
10+
import org.springframework.core.io.ResourceLoader;
11+
12+
import java.io.IOException;
13+
import java.nio.charset.StandardCharsets;
14+
15+
@Configuration
16+
@Data
17+
@AllArgsConstructor
18+
public class UserAgentConfig {
19+
20+
private ResourceLoader resourceLoader;
21+
private Gson gson;
22+
23+
@Bean
24+
public UserAgent[] userAgents() throws IOException {
25+
Resource resource = this.resourceLoader.getResource("classpath:mobile-user-agents.json");
26+
String json = resource.getContentAsString(StandardCharsets.UTF_8);
27+
28+
return this.gson.fromJson(json, UserAgent[].class);
29+
}
30+
31+
}

‎src/main/java/dev/manuelschuler/kleinanzeigenadsrenewer/helper/HttpClientHelper.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package dev.manuelschuler.kleinanzeigenadsrenewer.helper;
22

33
import dev.manuelschuler.kleinanzeigenadsrenewer.exceptions.RenewException;
4-
import generator.RandomUserAgentGenerator;
54
import lombok.SneakyThrows;
65
import okhttp3.OkHttpClient;
76
import okhttp3.Request;
@@ -13,14 +12,13 @@ private HttpClientHelper() {
1312
}
1413

1514
@SneakyThrows
16-
public static String doGetRequest(String url) {
17-
String randomUserAgent = RandomUserAgentGenerator.getNext();
15+
public static String doGetRequest(String url, String userAgent) {
1816
OkHttpClient client = new OkHttpClient().newBuilder().build();
1917
Request.Builder requestBuilder =
2018
new Request.Builder()
2119
.url(url)
2220
.method("GET", null)
23-
.addHeader("User-Agent", randomUserAgent);
21+
.addHeader("User-Agent", userAgent);
2422

2523
Request request = requestBuilder.build();
2624

‎src/main/java/dev/manuelschuler/kleinanzeigenadsrenewer/jobs/RenewJob.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import dev.manuelschuler.kleinanzeigenadsrenewer.model.ImapServer;
1212
import dev.manuelschuler.kleinanzeigenadsrenewer.service.AdMailScrapeService;
1313
import dev.manuelschuler.kleinanzeigenadsrenewer.service.ResultPageScrapeService;
14+
import dev.manuelschuler.kleinanzeigenadsrenewer.service.UserAgentService;
1415
import jakarta.mail.FetchProfile;
1516
import jakarta.mail.Folder;
1617
import jakarta.mail.Message;
@@ -34,6 +35,7 @@ public class RenewJob {
3435
private List<ImapServer> imapServers;
3536
private ImapFolderConfig imapFolderConfig;
3637
private RetryPolicy<Object> retryPolicy;
38+
private UserAgentService userAgentService;
3739

3840
private final Logger logger = LoggerFactory.getLogger(RenewJob.class);
3941

@@ -100,9 +102,10 @@ private void processImapServer(ImapServer imapServer) {
100102

101103
@SneakyThrows
102104
private void processAdMail(Message adMail, ImapServer imapServer) {
105+
String userAgent = this.userAgentService.getRandomUserAgent();
103106
AdMailScrapeService mailScraper = new AdMailScrapeService(adMail);
104107
String link = mailScraper.getRenewLink();
105-
String resultHtml = HttpClientHelper.doGetRequest(link);
108+
String resultHtml = HttpClientHelper.doGetRequest(link, userAgent);
106109

107110
ResultPageScrapeService resultPageScraper = new ResultPageScrapeService(resultHtml);
108111
Optional<String> errorMessage = resultPageScraper.getErrorMessage();

‎src/main/java/dev/manuelschuler/kleinanzeigenadsrenewer/model/UserAgent.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package dev.manuelschuler.kleinanzeigenadsrenewer.model;
2+
3+
import lombok.AllArgsConstructor;
4+
import lombok.Data;
5+
6+
@AllArgsConstructor
7+
@Data
8+
public class UserAgent {
9+
10+
private String ua;
11+
private Double pct;
12+
13+
}

‎src/main/java/dev/manuelschuler/kleinanzeigenadsrenewer/service/UserAgentService.java

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package dev.manuelschuler.kleinanzeigenadsrenewer.service;
2+
3+
import dev.manuelschuler.kleinanzeigenadsrenewer.model.UserAgent;
4+
import org.springframework.stereotype.Service;
5+
6+
import java.util.Arrays;
7+
import java.util.concurrent.ThreadLocalRandom;
8+
import java.util.concurrent.atomic.AtomicInteger;
9+
import java.util.concurrent.atomic.AtomicReference;
10+
11+
@Service
12+
public class UserAgentService {
13+
14+
private final UserAgent[] userAgents;
15+
private final AtomicReference<Double> totalWeight = new AtomicReference<>((double) 0);
16+
17+
public UserAgentService(UserAgent[] userAgents) {
18+
this.userAgents = userAgents;
19+
this.calculateTotalWeight();
20+
}
21+
22+
public String getRandomUserAgent() {
23+
double randomValue = ThreadLocalRandom.current().nextDouble(0, totalWeight.get());
24+
25+
AtomicInteger cumulativeWeight = new AtomicInteger();
26+
return Arrays.stream(userAgents)
27+
.filter(userAgent -> {
28+
cumulativeWeight.addAndGet((int) Math.round(userAgent.getPct()));
29+
return randomValue < cumulativeWeight.get();
30+
})
31+
.map(UserAgent::getUa)
32+
.findFirst()
33+
.orElse(null);
34+
}
35+
36+
private void calculateTotalWeight() {
37+
Arrays.stream(userAgents).map(UserAgent::getPct).forEach(pct -> totalWeight.updateAndGet(currentValue -> currentValue + pct));
38+
}
39+
40+
}

‎src/main/resources/mobile-user-agents.json

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
[
2+
{
3+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.3",
4+
"pct": 63.11
5+
},
6+
{
7+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_3_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3.1 Mobile/15E148 Safari/604.",
8+
"pct": 8.25
9+
},
10+
{
11+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_3_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/360.1.737798518 Mobile/15E148 Safari/604.",
12+
"pct": 5.83
13+
},
14+
{
15+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_3_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/134.0.6998.99 Mobile/15E148 Safari/604.",
16+
"pct": 4.85
17+
},
18+
{
19+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/27.0 Chrome/125.0.0.0 Mobile Safari/537.3",
20+
"pct": 3.88
21+
},
22+
{
23+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.",
24+
"pct": 3.4
25+
},
26+
{
27+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Mobile/15E148 Safari/604.",
28+
"pct": 1.94
29+
},
30+
{
31+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 18_1_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Mobile/15E148 Safari/604.",
32+
"pct": 1.94
33+
},
34+
{
35+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.3",
36+
"pct": 1.46
37+
},
38+
{
39+
"ua": "Mozilla/5.0 (Android 14; Mobile; rv:136.0) Gecko/136.0 Firefox/136.",
40+
"pct": 0.97
41+
},
42+
{
43+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Mobile Safari/537.3",
44+
"pct": 0.97
45+
},
46+
{
47+
"ua": "Mozilla/5.0 (Linux; Android 10; JNY-LX1; HMSCore 6.15.0.302) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.196 HuaweiBrowser/15.0.4.312 Mobile Safari/537.3",
48+
"pct": 0.97
49+
},
50+
{
51+
"ua": "Mozilla/5.0 (Android 15; Mobile; rv:136.0) Gecko/136.0 Firefox/136.",
52+
"pct": 0.49
53+
},
54+
{
55+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e YisouSpider/5.0 Safari/602.",
56+
"pct": 0.49
57+
},
58+
{
59+
"ua": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.",
60+
"pct": 0.49
61+
},
62+
{
63+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Mobile Safari/537.3",
64+
"pct": 0.49
65+
},
66+
{
67+
"ua": "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.3",
68+
"pct": 0.49
69+
}
70+
]

0 commit comments

Comments
 (0)