Skip to content

Commit 5622332

Browse files
cristianastorino-agilelabNicolò Bidotti
cristianastorino-agilelab
authored and
Nicolò Bidotti
committed
[WIT-682] MWAA SP does not validate provisioning request
# New features and improvements - Test coverage for MRs is now exported - Test coverage artifacts are now saved in the pipeline - CI optimizations # Bug fixes - Implemented proper validation for v1/validate endpoint - open-generator-cli uses a fixed version - Enabled akka.loglevel to INFO # Related issue Closes WIT-682 # Definition of Done for Feature/Hotfixes ## All Developments - [x] Feature was implemented as per the requirements - [x] If some code parts are complex they must be commented with code documentation - [x] CI/CD is successful - [x] Code coverage is not reduced, any new code is covered - [x] E2E/integration tests are successful (whether run locally or in CI/CD) - [x] If dependencies were changed, be sure that they will not impact the project, that their license is compatible, and that they introduce no vulnerabilities - [x] Documentation have been updated * Documentation has been updated with explanation of the new feature if it is user-facing (eg component now has additional setting) or it impacts him in some other way (eg optional field that becomes mandatory) * If it is a breaking change, we have documented it as such in the MR description in a "Breaking Changes" section - [x] Check that you are not affecting any existing environments with these changes, especially the Sandbox/Playground. This means that merging it to master and deploying it to these environments will not break them and **no manual operations that are not reported in the documentation will be needed** - [x] Check that nothing is out of order and nothing problematic is included in the changes (sensitive information, credentials, customer information or other intellectual property) as they could end up being public (we have Open Source SP already published and automatically mirrored) - [x] Security, Authentication and Authorization have been considered. No SQL injection, tokens handling, RBAC integration. Common security vulnerabilities identified and resolved ## API Development - [x] Semantic of API has been checked, it is comprehensible, meaningful, with no redundant information and user oriented - [x] Meaningful unit and integration tests are present - [x] API Parameters are checked and errors are handled - [x] Returned errors are meaningful to the user - [x] API contract has been defined and documented. Documentation means explaining the meaning of all fields and including at least one example - [x] Exceptions and errors are handled, without letting the underlying framework to respond with a generic Internal Server Error - [x] API Performance has been assessed and is good for real world use cases. Database accesses have been optimized. - [x] API is logging in compliance with audit standards, presence of sensitive information for GDPR has been assessed and removed or managed in case is needed ## DB Development - [x] The database schema is designed to accurately represent the data model and meet the requirements - [x] Tables, relationships, and constraints (e.g. primary keys, foreign keys, unique constraints) are defined appropriately and following a common convention - [x] Normalization principles are applied to eliminate data redundancy and ensure data integrity - [x] Schema semantic is meaningful - [x] Fields naming are following naming convention ( Ex. camelCase or snake_case) - [x] No fields with mixed behaviors and meaning. If a field is representing an enum, enum values are strongly mutually exclusive - [x] Data Types have been reviewed and they are a good fit for each field - [x] Indexes are defined to optimize query performance for frequently accessed data, paying attention to do not affect too much write path and the overall complexity - [x] Sensitive data is stored securely, encrypted if required, and access is restricted to authorized users - [x] Backup and restore procedures have been updated to introduce new or changed tables - [x] Migration scripts to upgrade and downgrade the database have been implemented and tested
1 parent 4271ae8 commit 5622332

30 files changed

+821
-218
lines changed

‎.gitlab-ci.yml

Lines changed: 23 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,7 @@ include:
66
ref: 'main'
77
file: 'common/witboost.downstream.gitlab-ci.yml'
88

9-
image: ubuntu:20.04
10-
11-
before_script:
12-
- apt-get update -yqq
13-
- DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
14-
- apt-get install -yqq openjdk-17-jdk-headless
15-
- apt-get install -yqq gpg
16-
- echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
17-
- mkdir -p /root/.gnupg
18-
- gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
19-
- chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
20-
- apt-get update -yqq
21-
- apt-get install -yqq sbt
9+
image: sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.9_9_1.9.7_2.13.12
2210

2311
variables:
2412
SBT_OPTS: "-Dsbt.global.base=sbt-cache/sbtboot -Dsbt.boot.directory=sbt-cache/boot -Dsbt.ivy.home=sbt-cache/ivy -Dsbt.ci=true"
@@ -33,7 +21,7 @@ cache:
3321

3422
stages:
3523
- setup
36-
- checkFormatting
24+
- check
3725
- test
3826
- build
3927
- package
@@ -52,12 +40,12 @@ setup:
5240
dotenv: vars.env
5341

5442
checkFormatting:
55-
stage: checkFormatting
43+
stage: check
5644
script:
5745
- 'sbt scalafmtSbtCheck scalafmtCheckAll'
5846

5947
witboost.helm.checks:
60-
stage: checkFormatting
48+
stage: check
6149
extends: .witboost.helm.base-job
6250
before_script: []
6351
cache: []
@@ -70,30 +58,35 @@ witboost.helm.checks:
7058
test:
7159
stage: test
7260
script:
73-
- apt-get install -yqq npm
74-
- npm install @openapitools/openapi-generator-cli -g
75-
- 'sbt clean generateCode coverage test multi-jvm:test coverageReport'
61+
- apt-get update -yqq && apt-get install -yqq npm
62+
- npm install @openapitools/openapi-generator-cli@2.7.0 -g
63+
- 'sbt clean generateCode coverage test coverageReport'
64+
coverage: '/Statement coverage[A-Za-z\.*]\s*:\s*([^%]+)/'
65+
artifacts:
66+
paths:
67+
- target/scala-2.13/scoverage-report/*
68+
- target/scala-2.13/coverage-report/*
69+
reports:
70+
coverage_report:
71+
coverage_format: cobertura
72+
path: 'target/scala-2.13/coverage-report/cobertura.xml'
7673

7774
build:
7875
services:
79-
- docker:19.03.12-dind
76+
- docker:24.0.5-dind
8077
stage: build
8178
variables:
8279
DOCKER_HOST: tcp://docker:2375
8380
script: |
84-
apt-get install -yqq curl
85-
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
86-
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu focal stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
87-
apt-get update -yqq
88-
apt-get install -yqq docker-ce-cli
89-
apt-get install -yqq npm
90-
npm install @openapitools/openapi-generator-cli -g
81+
apt-get update -yqq && apt-get install -yqq ca-certificates curl gnupg npm
82+
install -m 0755 -d /etc/apt/keyrings
83+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && chmod a+r /etc/apt/keyrings/docker.gpg
84+
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
85+
apt-get update -yqq && apt-get install -yqq docker-ce-cli
86+
npm install @openapitools/openapi-generator-cli@2.7.0 -g
9187
echo $VERSION
9288
docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
9389
sbt clean generateCode compile k8tyGitlabCIPublish docker:publish
94-
artifacts:
95-
reports:
96-
dotenv: vars.env
9790
9891
witboost.helm.deploy:
9992
stage: package

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
Designed by [Agile Lab](https://www.agilelab.it/), witboost is a versatile platform that addresses a wide range of sophisticated data engineering challenges. It enables businesses to discover, enhance, and productize their data, fostering the creation of automated data platforms that adhere to the highest standards of data governance. Want to know more about witboost? Check it out [here](https://www.agilelab.it/witboost) or [contact us!](https://www.agilelab.it/contacts).
88

9-
This repository is part of our Open Source projects meant to showcase witboost's integration capabilities and provide a "batteries-included" product.
9+
This repository is part of our [Starter Kit](https://github.com/agile-lab-dev/witboost-starter-kit) meant to showcase witboost's integration capabilities and provide a "batteries-included" product.
1010

1111
# MWAA Specific Provisioner
1212

‎aws-integration/src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/s3/gateway/S3GatewayError.scala

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,36 @@ import it.agilelab.datamesh.mwaaspecificprovisioner.s3.common.ShowableOps.showTh
77
trait S3GatewayError extends Exception with Product with Serializable
88

99
object S3GatewayError {
10-
final case class S3GatewayInitError(error: Throwable) extends S3GatewayError
11-
final case class ObjectExistsErr(bucket: String, key: String, error: Throwable) extends S3GatewayError
12-
final case class CreateFolderErr(bucket: String, folder: String, error: Throwable) extends S3GatewayError
13-
final case class CreateFileErr(bucket: String, key: String, error: Throwable) extends S3GatewayError
14-
final case class GetObjectContentErr(bucket: String, key: String, error: Throwable) extends S3GatewayError
15-
final case class ListObjectsErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError
16-
final case class ListVersionsErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError
17-
final case class ListDeleteMarkersErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError
18-
final case class CopyObjectErr(source: String, dest: String, error: Throwable) extends S3GatewayError
19-
final case class DeleteObjectErr(bucket: String, obj: String, error: Throwable) extends S3GatewayError
10+
final case class S3GatewayInitError(error: Throwable) extends S3GatewayError {
11+
override def getMessage: String = error.getMessage
12+
}
13+
final case class ObjectExistsErr(bucket: String, key: String, error: Throwable) extends S3GatewayError {
14+
override def getMessage: String = error.getMessage
15+
}
16+
final case class CreateFolderErr(bucket: String, folder: String, error: Throwable) extends S3GatewayError {
17+
override def getMessage: String = error.getMessage
18+
}
19+
final case class CreateFileErr(bucket: String, key: String, error: Throwable) extends S3GatewayError {
20+
override def getMessage: String = error.getMessage
21+
}
22+
final case class GetObjectContentErr(bucket: String, key: String, error: Throwable) extends S3GatewayError {
23+
override def getMessage: String = error.getMessage
24+
}
25+
final case class ListObjectsErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError {
26+
override def getMessage: String = error.getMessage
27+
}
28+
final case class ListVersionsErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError {
29+
override def getMessage: String = error.getMessage
30+
}
31+
final case class ListDeleteMarkersErr(bucket: String, prefix: String, error: Throwable) extends S3GatewayError {
32+
override def getMessage: String = error.getMessage
33+
}
34+
final case class CopyObjectErr(source: String, dest: String, error: Throwable) extends S3GatewayError {
35+
override def getMessage: String = error.getMessage
36+
}
37+
final case class DeleteObjectErr(bucket: String, obj: String, error: Throwable) extends S3GatewayError {
38+
override def getMessage: String = error.getMessage
39+
}
2040

2141
implicit val showS3GatewayError: Show[S3GatewayError] = Show.show {
2242
case e: S3GatewayInitError => show"S3GatewayInitError(${e.error})"

‎build.sbt

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,16 +114,15 @@ lazy val root = (project in file(".")).settings(
114114
name := "datamesh.mwaaspecificprovisioner",
115115
Test / parallelExecution := false,
116116
dockerBuildOptions ++= Seq("--network=host"),
117-
dockerBaseImage := "adoptopenjdk:11-jdk-hotspot",
117+
dockerBaseImage := "eclipse-temurin:17-jre-jammy",
118118
dockerUpdateLatest := true,
119119
daemonUser := "daemon",
120120
Docker / version := (ThisBuild / version).value,
121121
Docker / packageName :=
122122
s"registry.gitlab.com/agilefactory/witboost.mesh/provisioning/sandbox/witboost.mesh.provisioning.sandbox.mwaaspecificprovisioner",
123-
Docker / dockerExposedPorts := Seq(8080),
123+
Docker / dockerExposedPorts := Seq(8093),
124124
onChangedBuildSource := ReloadOnSourceChanges,
125125
scalafixOnCompile := true,
126126
semanticdbEnabled := true,
127127
semanticdbVersion := scalafixSemanticdb.revision
128-
).aggregate(clientGenerated).dependsOn(serverGenerated, awsIntegration).enablePlugins(JavaAppPackaging, MultiJvmPlugin)
129-
.configs(MultiJvm).setupBuildInfo
128+
).aggregate(clientGenerated).dependsOn(serverGenerated, awsIntegration).enablePlugins(JavaAppPackaging).setupBuildInfo

‎helm/files/application.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
akka {
2-
loglevel = "OFF"
2+
loglevel = "INFO"
33
actor.warn-about-java-serializer-usage = on
44
actor.allow-java-serialization = off
55
coordinated-shutdown.exit-jvm = on

‎project/plugins.sbt

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")
88

99
addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.9")
1010

11-
addSbtPlugin("com.typesafe.sbt" % "sbt-multi-jvm" % "0.4.0")
12-
1311
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
1412

1513
addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.34")

‎src/main/resources/reference.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
akka {
2-
loglevel = "OFF"
2+
loglevel = "INFO"
33
actor.warn-about-java-serializer-usage = on
44
actor.allow-java-serialization = off
55
coordinated-shutdown.exit-jvm = on

‎src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/api/intepreter/ProvisionerApiMarshallerImpl.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ class ProvisionerApiMarshallerImpl extends SpecificProvisionerApiMarshaller {
133133
))
134134
}
135135

136+
implicit def toEntityMarshallerReverseProvisioningRequest: ToEntityMarshaller[ReverseProvisioningRequest] =
137+
marshaller[ReverseProvisioningRequest]
138+
136139
implicit def fromEntityUnmarshallerReverseProvisioningRequest: FromEntityUnmarshaller[ReverseProvisioningRequest] =
137140
unmarshaller[ReverseProvisioningRequest]
138141

‎src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/api/intepreter/ProvisionerApiServiceImpl.scala

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,12 @@ package it.agilelab.datamesh.mwaaspecificprovisioner.api.intepreter
33
import akka.http.scaladsl.marshalling.ToEntityMarshaller
44
import akka.http.scaladsl.server.Route
55
import akka.http.scaladsl.unmarshalling.FromEntityUnmarshaller
6-
import cats.data.NonEmptyList
7-
import cats.implicits.toShow
6+
import cats.data.Validated.{Invalid, Valid}
87
import com.typesafe.scalalogging.LazyLogging
98
import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.{marshaller, unmarshaller}
109
import it.agilelab.datamesh.mwaaspecificprovisioner.api.SpecificProvisionerApiService
1110
import it.agilelab.datamesh.mwaaspecificprovisioner.model._
12-
import it.agilelab.datamesh.mwaaspecificprovisioner.mwaa.{MwaaManager, MwaaManagerError}
13-
import it.agilelab.datamesh.mwaaspecificprovisioner.s3.gateway.S3GatewayError
11+
import it.agilelab.datamesh.mwaaspecificprovisioner.mwaa.MwaaManager
1412

1513
class ProvisionerApiServiceImpl(mwaaManager: MwaaManager) extends SpecificProvisionerApiService with LazyLogging {
1614

@@ -59,23 +57,18 @@ class ProvisionerApiServiceImpl(mwaaManager: MwaaManager) extends SpecificProvis
5957
toEntityMarshallerRequestValidationError: ToEntityMarshaller[RequestValidationError],
6058
toEntityMarshallerSystemError: ToEntityMarshaller[SystemError],
6159
toEntityMarshallerProvisioningStatus: ToEntityMarshaller[ProvisioningStatus]
62-
): Route = ProvisioningRequestDescriptor(provisioningRequest.descriptor).flatMap(mwaaManager.executeProvision) match {
63-
case Left(e: S3GatewayError) =>
64-
logger.error(e.show)
65-
provision500(SystemError(e.show))
66-
case Left(e: MwaaManagerError) =>
67-
logger.error(e.errorMessage)
68-
provision500(SystemError(e.errorMessage))
69-
case Left(e: NonEmptyList[_]) =>
70-
logger.error(e.head.toString)
71-
provision400(RequestValidationError(e.toList.map(_.toString)))
72-
case Right(_) =>
73-
logger.info("OK")
74-
provision200(ProvisioningStatus(ProvisioningStatusEnums.StatusEnum.COMPLETED, "OK"))
75-
case other =>
76-
logger.error("Generic Error. Received {}", other)
77-
provision500(SystemError("Generic Error"))
78-
}
60+
): Route =
61+
try mwaaManager.executeProvision(provisioningRequest.descriptor) match {
62+
case Valid(_) => provision200(ProvisioningStatus(ProvisioningStatusEnums.StatusEnum.COMPLETED, "OK"))
63+
case Invalid(e) => provision400(RequestValidationError(e.toList.map(_.errorMessage)))
64+
}
65+
catch {
66+
case t: Throwable =>
67+
logger.error(s"Exception in provision", t)
68+
provision500(SystemError(
69+
s"An unexpected error occurred while processing the request. Please try again and if the problem persists contact the platform team. Details: ${t.getMessage}"
70+
))
71+
}
7972

8073
/** Code: 200, Message: It synchronously returns the request result, DataType: String
8174
* Code: 400, Message: Invalid input, DataType: RequestValidationError
@@ -85,7 +78,20 @@ class ProvisionerApiServiceImpl(mwaaManager: MwaaManager) extends SpecificProvis
8578
contexts: Seq[(String, String)],
8679
toEntityMarshallerSystemError: ToEntityMarshaller[SystemError],
8780
toEntityMarshallerValidationResult: ToEntityMarshaller[ValidationResult]
88-
): Route = validate200(ValidationResult(valid = true))
81+
): Route =
82+
try mwaaManager.executeValidation(provisioningRequest.descriptor) match {
83+
case Valid(_) => validate200(ValidationResult(valid = true))
84+
case Invalid(e) =>
85+
val errors = e.map(_.errorMessage).toList
86+
validate200(ValidationResult(valid = false, error = Some(ValidationError(errors))))
87+
}
88+
catch {
89+
case t: Throwable =>
90+
logger.error(s"Exception in validate", t)
91+
validate500(SystemError(
92+
s"An unexpected error occurred while processing the request. Please try again and if the problem persists contact the platform team. Details: ${t.getMessage}"
93+
))
94+
}
8995

9096
/** Code: 200, Message: It synchronously returns the request result, DataType: ProvisioningStatus
9197
* Code: 202, Message: If successful returns a provisioning deployment task token that can be used for polling the request status, DataType: String
@@ -98,22 +104,16 @@ class ProvisionerApiServiceImpl(mwaaManager: MwaaManager) extends SpecificProvis
98104
toEntityMarshallerSystemError: ToEntityMarshaller[SystemError],
99105
toEntityMarshallerProvisioningStatus: ToEntityMarshaller[ProvisioningStatus]
100106
): Route =
101-
ProvisioningRequestDescriptor(provisioningRequest.descriptor).flatMap(mwaaManager.executeUnprovision) match {
102-
case Left(e: S3GatewayError) =>
103-
logger.error(e.show)
104-
provision500(SystemError(e.show))
105-
case Left(e: MwaaManagerError) =>
106-
logger.error(e.errorMessage)
107-
provision500(SystemError(e.errorMessage))
108-
case Left(e: NonEmptyList[_]) =>
109-
logger.error(e.head.toString)
110-
provision400(RequestValidationError(e.toList.map(_.toString)))
111-
case Right(_) =>
112-
logger.info("OK")
113-
provision200(ProvisioningStatus(ProvisioningStatusEnums.StatusEnum.COMPLETED, "OK"))
114-
case other =>
115-
logger.error("Generic Error. Received {}", other)
116-
provision500(SystemError("Generic Error"))
107+
try mwaaManager.executeUnprovision(provisioningRequest.descriptor) match {
108+
case Valid(_) => unprovision200(ProvisioningStatus(ProvisioningStatusEnums.StatusEnum.COMPLETED, "OK"))
109+
case Invalid(e) => unprovision400(RequestValidationError(e.toList.map(_.errorMessage)))
110+
}
111+
catch {
112+
case t: Throwable =>
113+
logger.error(s"Exception in unprovision", t)
114+
unprovision500(SystemError(
115+
s"An unexpected error occurred while processing the request. Please try again and if the problem persists contact the platform team. Details: ${t.getMessage}"
116+
))
117117
}
118118

119119
/** Code: 200, Message: It synchronously returns the access request response, DataType: ProvisioningStatus
@@ -126,7 +126,7 @@ class ProvisionerApiServiceImpl(mwaaManager: MwaaManager) extends SpecificProvis
126126
toEntityMarshallerRequestValidationError: ToEntityMarshaller[RequestValidationError],
127127
toEntityMarshallerSystemError: ToEntityMarshaller[SystemError],
128128
toEntityMarshallerProvisioningStatus: ToEntityMarshaller[ProvisioningStatus]
129-
): Route = updateacl200(ProvisioningStatus(ProvisioningStatusEnums.StatusEnum.COMPLETED, "OK"))
129+
): Route = updateacl500(NotImplementedError)
130130

131131
/** Code: 202, Message: It returns a token that can be used for polling the async validation operation status and results, DataType: String
132132
* Code: 400, Message: Invalid input, DataType: RequestValidationError

‎src/main/scala/it/agilelab/datamesh/mwaaspecificprovisioner/common/Constants.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,5 @@ object Constants {
1515
val SOURCE_DAG_PATH_FIELD = "sourcePath"
1616
val BUCKET_NAME_FIELD = "bucketName"
1717
val DAG_NAME_FIELD = "dagName"
18+
val SCHEDULE_CRON_FIELD = "scheduleCron"
1819
}

0 commit comments

Comments
 (0)