Skip to content

Commit dd653ec

Browse files
KarimGlKarimGl
authored
feat(server): zip and index scenario execution report (#208)
1 parent d3db2ec commit dd653ec

File tree

22 files changed

+715
-15
lines changed

22 files changed

+715
-15
lines changed

‎chutney/packaging/local-dev/src/main/resources/application.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ chutney:
6262
configuration-folder: ${chutney.configuration-folder}/environment
6363
jira:
6464
configuration-folder: ${chutney.configuration-folder}/jira
65-
65+
index-folder: .chutney/index
6666
server:
6767
editions:
6868
ttl:

‎chutney/pom.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@
7474
<!-- test dependencies versions -->
7575
<jqwik.version>1.9.1</jqwik.version>
7676
<wiremock.version>3.9.1</wiremock.version>
77+
<lucene.version>9.12.0</lucene.version>
7778
</properties>
7879

7980
<dependencies>
@@ -261,6 +262,21 @@
261262
<artifactId>liquibase-core</artifactId>
262263
<version>${liquibase.version}</version>
263264
</dependency>
265+
<dependency>
266+
<groupId>org.apache.lucene</groupId>
267+
<artifactId>lucene-core</artifactId>
268+
<version>${lucene.version}</version>
269+
</dependency>
270+
<dependency>
271+
<groupId>org.apache.lucene</groupId>
272+
<artifactId>lucene-analysis-common</artifactId>
273+
<version>${lucene.version}</version>
274+
</dependency>
275+
<dependency>
276+
<groupId>org.apache.lucene</groupId>
277+
<artifactId>lucene-queryparser</artifactId>
278+
<version>${lucene.version}</version>
279+
</dependency>
264280
<!-- Test dependencies -->
265281
<dependency>
266282
<groupId>org.springframework.boot</groupId>

‎chutney/server-core/src/main/java/com/chutneytesting/server/core/domain/execution/history/ExecutionHistoryRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public interface ExecutionHistoryRepository {
4747
*/
4848
ExecutionHistory.Execution getExecution(String scenarioId, Long reportId) throws ReportNotFoundException;
4949

50-
List<ExecutionHistory.ExecutionSummary> getExecutionReportMatchQuery(String query);
50+
List<ExecutionHistory.ExecutionSummary> getExecutionReportMatchKeyword(String query);
5151

5252
/**
5353
* Override a previously stored {@link ExecutionHistory.Execution}.

‎chutney/server/pom.xml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@
5050
<groupId>org.springframework.boot</groupId>
5151
<artifactId>spring-boot-starter-data-jpa</artifactId>
5252
</dependency>
53+
<dependency>
54+
<groupId>org.springframework.boot</groupId>
55+
<artifactId>spring-boot-starter-aop</artifactId>
56+
</dependency>
5357
<dependency>
5458
<groupId>org.springframework.boot</groupId>
5559
<artifactId>spring-boot-starter-webflux</artifactId>
@@ -181,6 +185,19 @@
181185
<scope>runtime</scope>
182186
</dependency>
183187

188+
<dependency>
189+
<groupId>org.apache.lucene</groupId>
190+
<artifactId>lucene-core</artifactId>
191+
</dependency>
192+
<dependency>
193+
<groupId>org.apache.lucene</groupId>
194+
<artifactId>lucene-analysis-common</artifactId>
195+
</dependency>
196+
<dependency>
197+
<groupId>org.apache.lucene</groupId>
198+
<artifactId>lucene-queryparser</artifactId>
199+
</dependency>
200+
184201
<!-- Test dependencies -->
185202
<dependency>
186203
<groupId>org.springframework.boot</groupId>

‎chutney/server/src/main/java/com/chutneytesting/ServerConfiguration.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,13 @@
6161
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
6262
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
6363
import org.springframework.context.annotation.Bean;
64+
import org.springframework.context.annotation.EnableAspectJAutoProxy;
6465
import org.springframework.core.task.TaskExecutor;
6566
import org.springframework.core.task.support.ExecutorServiceAdapter;
6667
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
6768

6869
@SpringBootApplication(exclude = {LiquibaseAutoConfiguration.class, ActiveMQAutoConfiguration.class, MongoAutoConfiguration.class})
70+
@EnableAspectJAutoProxy
6971
public class ServerConfiguration {
7072

7173
private static final Logger LOGGER = LoggerFactory.getLogger(ServerConfiguration.class);

‎chutney/server/src/main/java/com/chutneytesting/admin/api/DatabaseManagementController.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public class DatabaseManagementController {
4040
@PreAuthorize("hasAuthority('ADMIN_ACCESS')")
4141
@GetMapping(path = "/execution", produces = MediaType.APPLICATION_JSON_VALUE)
4242
public List<ExecutionSummaryDto> getExecutionReportMatchQuery(@QueryParam("query") String query) {
43-
return executionHistoryRepository.getExecutionReportMatchQuery(query).stream().map(ExecutionSummaryDto::toDto).toList();
43+
return executionHistoryRepository.getExecutionReportMatchKeyword(query).stream().map(ExecutionSummaryDto::toDto).toList();
4444
}
4545

4646
@PreAuthorize("hasAuthority('ADMIN_ACCESS')")

‎chutney/server/src/main/java/com/chutneytesting/execution/infra/aop/ScenarioExecutionReportIndexingAspect.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2017-2024 Enedis
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*
6+
*/
7+
8+
package com.chutneytesting.execution.infra.aop;
9+
10+
import com.chutneytesting.execution.infra.storage.DatabaseExecutionJpaRepository;
11+
import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionEntity;
12+
import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity;
13+
import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository;
14+
import com.chutneytesting.scenario.infra.jpa.ScenarioEntity;
15+
import java.util.List;
16+
import java.util.Set;
17+
import java.util.stream.Collectors;
18+
import org.aspectj.lang.annotation.After;
19+
import org.aspectj.lang.annotation.Aspect;
20+
import org.springframework.stereotype.Component;
21+
22+
@Aspect
23+
@Component
24+
public class ScenarioExecutionReportIndexingAspect {
25+
private final ScenarioExecutionReportIndexRepository reportIndexRepository;
26+
private final DatabaseExecutionJpaRepository scenarioExecutionRepository;
27+
28+
public ScenarioExecutionReportIndexingAspect(ScenarioExecutionReportIndexRepository reportIndexRepository, DatabaseExecutionJpaRepository scenarioExecutionRepository) {
29+
this.reportIndexRepository = reportIndexRepository;
30+
this.scenarioExecutionRepository = scenarioExecutionRepository;
31+
}
32+
33+
@After("execution(* com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository.save(..)) && args(reportEntity)")
34+
public void index(ScenarioExecutionReportEntity reportEntity) {
35+
if (reportEntity.status().isFinal()){
36+
reportIndexRepository.save(reportEntity);
37+
}
38+
}
39+
40+
@After("execution(* com.chutneytesting.scenario.infra.raw.ScenarioJpaRepository.save(..)) && args(scenario)")
41+
public void deleteDeactivatedScenarioExecutions(ScenarioEntity scenario) {
42+
if (!scenario.isActivated()){
43+
List<ScenarioExecutionEntity> executions = scenarioExecutionRepository.findAllByScenarioId(String.valueOf(scenario.getId()));
44+
reportIndexRepository.deleteAllById(executions.stream().map(ScenarioExecutionEntity::getId).collect(Collectors.toSet()));
45+
}
46+
47+
}
48+
49+
@After("execution(* com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository.deleteAllById(..)) && args(scenarioExecutionIds)")
50+
public void deleteById(Set<Long> scenarioExecutionIds) {
51+
reportIndexRepository.deleteAllById(scenarioExecutionIds);
52+
}
53+
}

‎chutney/server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* SPDX-FileCopyrightText: 2017-2024 Enedis
3+
*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*
6+
*/
7+
8+
package com.chutneytesting.execution.infra.migration;
9+
10+
import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.SCENARIO_EXECUTION_REPORT;
11+
import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.WHAT;
12+
13+
import com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository;
14+
import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity;
15+
import com.chutneytesting.index.infra.IndexRepository;
16+
import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository;
17+
import jakarta.persistence.EntityManager;
18+
import java.util.List;
19+
import org.apache.lucene.index.Term;
20+
import org.apache.lucene.search.Query;
21+
import org.apache.lucene.search.TermQuery;
22+
import org.slf4j.Logger;
23+
import org.slf4j.LoggerFactory;
24+
import org.springframework.boot.CommandLineRunner;
25+
import org.springframework.data.domain.PageRequest;
26+
import org.springframework.data.domain.Pageable;
27+
import org.springframework.data.domain.Slice;
28+
import org.springframework.stereotype.Component;
29+
import org.springframework.transaction.annotation.Transactional;
30+
31+
@Component
32+
public class ZipReportMigration implements CommandLineRunner {
33+
34+
35+
private final ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository;
36+
private final ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository;
37+
private final IndexRepository indexRepository;
38+
private final EntityManager entityManager;
39+
private static final Logger LOGGER = LoggerFactory.getLogger(ZipReportMigration.class);
40+
41+
42+
public ZipReportMigration(ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository, ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository, IndexRepository indexRepository, EntityManager entityManager) {
43+
this.scenarioExecutionReportIndexRepository = scenarioExecutionReportIndexRepository;
44+
this.scenarioExecutionReportJpaRepository = scenarioExecutionReportJpaRepository;
45+
this.indexRepository = indexRepository;
46+
this.entityManager = entityManager;
47+
}
48+
49+
@Override
50+
@Transactional
51+
public void run(String... args) {
52+
if (isMigrationDone()) {
53+
LOGGER.info("Report compression & indexing already done, skipping...");
54+
return;
55+
}
56+
PageRequest firstPage = PageRequest.of(0, 10);
57+
int count = 0;
58+
compressAndIndex(firstPage, count);
59+
}
60+
61+
private void compressAndIndex(Pageable pageable, int previousCount) {
62+
Slice<ScenarioExecutionReportEntity> slice = scenarioExecutionReportJpaRepository.findAll(pageable);
63+
List<ScenarioExecutionReportEntity> reports = slice.getContent();
64+
65+
compressAndSaveInDb(reports);
66+
index(reports);
67+
68+
int count = previousCount + slice.getNumberOfElements();
69+
if (slice.hasNext()) {
70+
compressAndIndex(slice.nextPageable(), count);
71+
} else {
72+
LOGGER.info("{} report(s) successfully compressed and indexed", count);
73+
}
74+
}
75+
76+
private void compressAndSaveInDb(List<ScenarioExecutionReportEntity> reportsInDb) {
77+
// calling scenarioExecutionReportJpaRepository find() and then save() doesn't call ReportConverter
78+
// ReportConverter will be called by entityManager update. So compression will be done
79+
reportsInDb.forEach(report -> {
80+
entityManager.createQuery(
81+
"UPDATE SCENARIO_EXECUTIONS_REPORTS SET report = :report WHERE id = :id")
82+
.setParameter("report", report.getReport())
83+
.setParameter("id", report.scenarioExecutionId())
84+
.executeUpdate();
85+
});
86+
}
87+
88+
private void index(List<ScenarioExecutionReportEntity> reportsInDb) {
89+
scenarioExecutionReportIndexRepository.saveAll(reportsInDb);
90+
}
91+
92+
private boolean isMigrationDone() {
93+
Query whatQuery = new TermQuery(new Term(WHAT, SCENARIO_EXECUTION_REPORT));
94+
int indexedReports = indexRepository.count(whatQuery);
95+
return indexedReports > 0;
96+
}
97+
}

‎chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionHistoryRepository.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.chutneytesting.campaign.infra.jpa.CampaignExecutionEntity;
1717
import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionEntity;
1818
import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity;
19+
import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository;
1920
import com.chutneytesting.server.core.domain.dataset.DataSet;
2021
import com.chutneytesting.server.core.domain.execution.history.ExecutionHistory.DetachedExecution;
2122
import com.chutneytesting.server.core.domain.execution.history.ExecutionHistory.Execution;
@@ -51,6 +52,7 @@ class DatabaseExecutionHistoryRepository implements ExecutionHistoryRepository {
5152
private final CampaignJpaRepository campaignJpaRepository;
5253
private final CampaignExecutionJpaRepository campaignExecutionJpaRepository;
5354
private final TestCaseRepository testCaseRepository;
55+
private final ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository;
5456
private final ObjectMapper objectMapper;
5557
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseExecutionHistoryRepository.class);
5658

@@ -60,12 +62,14 @@ class DatabaseExecutionHistoryRepository implements ExecutionHistoryRepository {
6062
ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository,
6163
CampaignJpaRepository campaignJpaRepository, TestCaseRepository testCaseRepository,
6264
CampaignExecutionJpaRepository campaignExecutionJpaRepository,
65+
ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository,
6366
@Qualifier("reportObjectMapper") ObjectMapper objectMapper) {
6467
this.scenarioExecutionsJpaRepository = scenarioExecutionsJpaRepository;
6568
this.scenarioExecutionReportJpaRepository = scenarioExecutionReportJpaRepository;
6669
this.campaignJpaRepository = campaignJpaRepository;
6770
this.testCaseRepository = testCaseRepository;
6871
this.campaignExecutionJpaRepository = campaignExecutionJpaRepository;
72+
this.scenarioExecutionReportIndexRepository = scenarioExecutionReportIndexRepository;
6973
this.objectMapper = objectMapper;
7074
}
7175

@@ -128,7 +132,8 @@ public Execution store(String scenarioId, DetachedExecution detachedExecution) t
128132
scenarioExecution.forCampaignExecution(campaignExecution.get());
129133
}
130134
scenarioExecution = scenarioExecutionsJpaRepository.save(scenarioExecution);
131-
scenarioExecutionReportJpaRepository.save(new ScenarioExecutionReportEntity(scenarioExecution, detachedExecution.report()));
135+
ScenarioExecutionReportEntity reportEntity = new ScenarioExecutionReportEntity(scenarioExecution, detachedExecution.report());
136+
scenarioExecutionReportJpaRepository.save(reportEntity);
132137
Execution execution = detachedExecution.attach(scenarioExecution.id(), scenarioId);
133138
return ImmutableExecutionHistory.Execution.builder().from(execution).build();
134139
}
@@ -146,9 +151,10 @@ public Execution getExecution(String scenarioId, Long reportId) throws ReportNot
146151
}
147152

148153
@Override
149-
public List<ExecutionSummary> getExecutionReportMatchQuery(String query) {
154+
public List<ExecutionSummary> getExecutionReportMatchKeyword(String keyword) {
155+
List<Long> matchedReportsIds = scenarioExecutionReportIndexRepository.idsByKeywordInReport(keyword);
150156
return scenarioExecutionsJpaRepository
151-
.getExecutionReportMatchQuery(query)
157+
.getExecutionReportByIds(matchedReportsIds)
152158
.stream()
153159
.map(this::scenarioExecutionToExecutionSummary)
154160
.toList();

‎chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionJpaRepository.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,8 @@ ELSE MAX(CASE WHEN se.status != 'NOT_EXECUTED' THEN se.id END)
5353
inner join ser.scenarioExecution se
5454
where s.activated = true
5555
and cast(s.id as string) = se.scenarioId
56-
and ser.report like '%' || :query || '%'
56+
and ser.scenarioExecutionId in (:executionsIds)
5757
order by se.id desc
58-
limit 100
5958
""")
60-
List<ScenarioExecutionEntity> getExecutionReportMatchQuery(@Param("query") String query);
59+
List<ScenarioExecutionEntity> getExecutionReportByIds(@Param("executionsIds") List<Long> executionsIds);
6160
}

0 commit comments

Comments
 (0)