From 25436a0188a72cd13f5a80721381138cbb00ea97 Mon Sep 17 00:00:00 2001 From: Nathan Date: Wed, 2 Jul 2025 14:58:27 -0700 Subject: [PATCH] feat: S3 support --- gradle/elide.versions.toml | 5 + packages/graalvm/build.gradle.kts | 4 + .../elide/runtime/intrinsics/js/s3/S3.kt | 132 ++++ .../runtime/intrinsics/js/s3/S3Module.kt | 436 +++++++++++++ .../runtime/intrinsics/s3/S3ModuleTest.kt | 580 ++++++++++++++++++ 5 files changed, 1157 insertions(+) create mode 100644 packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3.kt create mode 100644 packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3Module.kt create mode 100644 packages/graalvm/src/test/kotlin/elide/runtime/intrinsics/s3/S3ModuleTest.kt diff --git a/gradle/elide.versions.toml b/gradle/elide.versions.toml index b8ec58dfd6..75b8866817 100644 --- a/gradle/elide.versions.toml +++ b/gradle/elide.versions.toml @@ -27,6 +27,7 @@ auto-common = "1.2.2" auto-factory = "1.1.0" auto-service = "1.1.1" auto-value = "1.11.0" +aws-sdk = "2.31.74" bouncycastle = "1.81" brotli = "0.1.2" brotli4j = "1.18.0" @@ -194,6 +195,7 @@ larray = "0.2.1" lettuce = "6.2.5.RELEASE" lmaxDisruptor = "4.0.0" lmaxDisruptorProxy = "3.1.1" +locals3 = "1.25" logback = "1.5.18" lz4 = "1.3.0" markdown = "0.7.3" @@ -363,6 +365,8 @@ asm-core = { group = "org.ow2.asm", name = "asm", version.ref = "asm" } asm-tree = { group = "org.ow2.asm", name = "asm-tree", version.ref = "asm" } assertk = { group = "com.willowtreeapps.assertk", name = "assertk", version.ref = "assertk" } autoService-ksp = { module = "dev.zacsweers.autoservice:auto-service-ksp", version = "1.1.0" } +aws-sdk-bom = { group = "software.amazon.awssdk", name = "bom", version.ref = "aws-sdk" } +aws-sdk-s3 = { group = "software.amazon.awssdk", name = "s3" } bouncycastle = { group = "org.bouncycastle", name = "bcprov-jdk18on", version.ref = "bouncycastle" } bouncycastle-pkix = { group = "org.bouncycastle", name = "bcpkix-jdk18on", version.ref = "bouncycastle" } bouncycastle-tls = { group = "org.bouncycastle", name = "bctls-jdk18on", version.ref = "bouncycastle" } @@ -706,6 +710,7 @@ larray-mmap = { group = "org.xerial.larray", name = "larray-mmap", version.ref = lettuce-core = { group = "io.lettuce", name = "lettuce-core", version.ref = "lettuce" } lmax-disruptor-core = { group = "com.lmax", name = "disruptor", version.ref = "lmaxDisruptor" } lmax-disruptor-proxy = { group = "com.lmax", name = "disruptor-proxy", version.ref = "lmaxDisruptorProxy" } +locals3 = { group = "io.github.robothy", name = "local-s3-rest", version.ref = "locals3" } logback = { group = "ch.qos.logback", name = "logback-classic", version.ref = "logback" } logback-core = { group = "ch.qos.logback", name = "logback-core", version.ref = "logback" } lz4 = { group = "net.jpountz.lz4", name = "lz4", version.ref = "lz4" } diff --git a/packages/graalvm/build.gradle.kts b/packages/graalvm/build.gradle.kts index dd1bec7caf..8d89e29557 100644 --- a/packages/graalvm/build.gradle.kts +++ b/packages/graalvm/build.gradle.kts @@ -555,10 +555,14 @@ dependencies { api(libs.graalvm.polyglot.js.community) } + implementation(platform(libs.aws.sdk.bom)) + implementation(libs.aws.sdk.s3) + // Testing testApi(project(":packages:engine", configuration = "testInternals")) testApi(libs.graalvm.truffle.api) testApi(libs.graalvm.truffle.runtime) + testImplementation(libs.locals3) testImplementation(libs.jackson.core) testImplementation(libs.jackson.databind) testImplementation(libs.jackson.module.kotlin) diff --git a/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3.kt b/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3.kt new file mode 100644 index 0000000000..9584686fe9 --- /dev/null +++ b/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3.kt @@ -0,0 +1,132 @@ +package elide.runtime.intrinsics.js.s3 + +import org.graalvm.polyglot.Value +import org.graalvm.polyglot.proxy.ProxyObject +import elide.runtime.gvm.js.JsError +import elide.runtime.intrinsics.js.Blob +import elide.runtime.intrinsics.js.JsPromise +import elide.runtime.intrinsics.js.ReadableStream +import elide.runtime.node.buffer.GuestBytes +import elide.vm.annotations.Polyglot + +private const val ACCESS_KEY_ID = "accessKeyId" +private const val SECRET_ACCESS_KEY = "secretAccessKey" +private const val BUCKET = "bucket" +private const val SESSION_TOKEN = "sessionToken" +private const val ACL = "acl" +private const val ENDPOINT = "endpoint" +private const val REGION = "region" +private const val VIRTUAL_HOSTED_STYLE = "virtualHostedStyle" + +/** + * # S3 + * + * Facade for calling out to AWS S3 SDK. Meant to be used to provide an S3 + * API from within the Elide runtime. + * + * It consists of two major objects, the S3Client which tracks the configuration for authentication and how to connect + * and the S3File which represents a file in S3 that can be read, written to, or presigned. + * + *   + * + * ## Usage + * + * In the following example, the S3Client will connect to Backblaze B2 (an S3 compatible service) + * using a path style endpoint. If the endpoint is not specified, by default it will use AWS S3. + * If region is unspecified it will use us-east-1. + * It will use path style URLs by default. + * + * Please look at AWS's S3 documentation or the documentation for whatever S3 compatible API is being used for more + * information on what these options mean. + * + * ```javascript + * const { S3Client } = require("elide:s3"); + * const client = new S3Client({ + * // required options + * accessKeyId: "...", + * secretAccessKey: "...", + * bucket: "my-bucket", + * // optional options + * sessionToken: "...", + * acl: "public-read", + * endpoint: "https://s3.us-east-005.backblazeb2.com/my-bucket", + * region: "us-east-005", + * virtualHostedStyle: false + * }); + * const file = client.file("test.txt"); + * await file.write("This is a test", { type: "text/plain" }); + * console.assert(await file.text() === "This is a test"); + * ``` + * + * Write method takes either a String or any array type object and returns a promise + * containing the content length. + * + * Read methods include: text, json, bytes, and arrayBuffer. Which are self-explanatory. + * + * Presign allows one to presign a URL to be passed to the client so the client can do the + * actual action, minimizing a redundant data transfer, a common pattern with S3 usage. + * eg + * ``` + * file.presign({ method: "GET", expiresIn: 3600 }); // expiresIn is in seconds + * ``` + * + * `delete` and `unlink` are identical. Both allow the user to delete a file from S3. + * + * Finally, `stat` returns the metadata and objects in the HTTP header in a JSON format. It + * is equivalent to the "HeadObject" action. + */ +public interface S3Client : ProxyObject { + @Polyglot public fun file(path: String): S3File +} + +public interface S3File : ProxyObject { + // write methods + @Polyglot public fun write(data: String, contentType: String?): JsPromise + @Polyglot public fun write(data: GuestBytes, contentType: String?): JsPromise + @Polyglot public fun write(data: Blob, contentType: String?): JsPromise + @Polyglot public fun write(data: ReadableStream, contentType: String?): JsPromise + + // read methods + @Polyglot public fun text(): JsPromise + @Polyglot public fun json(): JsPromise + @Polyglot public fun bytes(): JsPromise + @Polyglot public fun arrayBuffer(): JsPromise + + @Polyglot public fun presign(method: String, duration: Long): String + @Polyglot public fun delete(): JsPromise + + @Polyglot public fun exists(): JsPromise + @Polyglot public fun stat(): JsPromise +} + +@JvmRecord public data class S3ClientConstructorOptions( + @get:Polyglot val accessKeyId: String, + @get:Polyglot val secretAccessKey: String, + @get:Polyglot val bucket: String, + @get:Polyglot val sessionToken: String?, + @get:Polyglot val acl: String?, + @get:Polyglot val endpoint: String?, + @get:Polyglot val region: String?, + @get:Polyglot val virtualHostedStyle: Boolean? +) { + public companion object { + @JvmStatic public fun from(value: Value): S3ClientConstructorOptions { + return S3ClientConstructorOptions( + // required + accessKeyId = value.getMember(ACCESS_KEY_ID).asString() + ?: throw JsError.typeError("S3Client constructor missing accessKeyId"), + secretAccessKey = value.getMember(SECRET_ACCESS_KEY).asString() + ?: throw JsError.typeError("S3Client constructor missing secretAccessKey"), + bucket = value.getMember(BUCKET).asString() + ?: throw JsError.typeError("S3Client constructor missing bucket"), + + // optional + sessionToken = value.getMember(SESSION_TOKEN)?.asString(), + acl = value.getMember(ACL)?.asString(), + endpoint = value.getMember(ENDPOINT)?.asString(), + region = value.getMember(REGION)?.asString(), + virtualHostedStyle = value.getMember(VIRTUAL_HOSTED_STYLE)?.asBoolean() + ) + } + } +} diff --git a/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3Module.kt b/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3Module.kt new file mode 100644 index 0000000000..2c1092f5bc --- /dev/null +++ b/packages/graalvm/src/main/kotlin/elide/runtime/intrinsics/js/s3/S3Module.kt @@ -0,0 +1,436 @@ +package elide.runtime.intrinsics.js.s3 + +import org.graalvm.polyglot.Context +import org.graalvm.polyglot.Value +import org.graalvm.polyglot.proxy.ProxyArray +import org.graalvm.polyglot.proxy.ProxyExecutable +import org.graalvm.polyglot.proxy.ProxyInstantiable +import org.graalvm.polyglot.proxy.ProxyObject +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.core.ResponseBytes +import software.amazon.awssdk.core.async.AsyncRequestBody +import software.amazon.awssdk.core.async.AsyncResponseTransformer +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3AsyncClient +import software.amazon.awssdk.services.s3.model.* +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.DeleteObjectPresignRequest +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest +import software.amazon.awssdk.services.s3.presigner.model.HeadObjectPresignRequest +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest +import java.net.URI +import java.net.URLConnection +import java.nio.charset.StandardCharsets +import java.time.Duration +import java.util.concurrent.CompletionException +import elide.annotations.Singleton +import elide.runtime.exec.GuestExecutorProvider +import elide.runtime.gvm.api.Intrinsic +import elide.runtime.gvm.internals.intrinsics.js.AbstractJsIntrinsic +import elide.runtime.gvm.internals.intrinsics.js.ArrayBufferViewType +import elide.runtime.gvm.internals.intrinsics.js.ArrayBufferViews +import elide.runtime.gvm.js.JsError +import elide.runtime.gvm.loader.ModuleInfo +import elide.runtime.gvm.loader.ModuleRegistry +import elide.runtime.intrinsics.GuestIntrinsic +import elide.runtime.intrinsics.js.Blob +import elide.runtime.intrinsics.js.JsPromise +import elide.runtime.intrinsics.js.ReadableStream +import elide.runtime.node.buffer.GuestBufferView +import elide.runtime.node.buffer.GuestBytes +import elide.vm.annotations.Polyglot + + +private const val S3_MODULE_NAME = "s3" +private const val S3_CLIENT_SYMBOL = "S3Client" + +private const val CONTENT_TYPE_DEFAULT = "application/octet-stream" + +private const val DEFAULT_PRESIGN_METHOD = "GET" +private const val DEFAULT_PRESIGN_DURATION: Long = 86400 // 24 hours + +@Singleton +@Intrinsic +internal class S3Module (private val guestExec: GuestExecutorProvider) : AbstractJsIntrinsic() { + private val constructor = ProxyInstantiable { arguments -> + val options = arguments.getOrNull(0) + ?: throw JsError.typeError("No options passed!") + val config = S3ClientConstructorOptions.from(options) + S3ClientProxy(this.guestExec, config) + } + + override fun install(bindings: GuestIntrinsic.MutableIntrinsicBindings) { + ModuleRegistry.deferred(ModuleInfo.of(S3_MODULE_NAME)) { + ProxyObject.fromMap(mapOf( + S3_CLIENT_SYMBOL to constructor, + )) + } + } +} + +public class S3ClientProxy (private val guestExec: GuestExecutorProvider, + private val config: S3ClientConstructorOptions) : S3Client { + private val s3Client: S3AsyncClient + private val credentialsProvider: AwsCredentialsProvider + private val clientRegion: Region? + + // This map represents the JavaScript object representation of the class to the guest environment + // i.e. it is a map from strings to methods (in this case) + private val jsObj: Map = mapOf( + "file" to ProxyExecutable { args -> + val path = args.getOrNull(0)?.asString() ?: + throw JsError.typeError("S3Client.file() requires one string argument") + file(path) + } + ) + + init { + val credentials = AwsBasicCredentials.create(config.accessKeyId, config.secretAccessKey) + this.credentialsProvider = StaticCredentialsProvider.create(credentials) + this.clientRegion = when { + config.region != null -> Region.of(config.region) + config.endpoint == null -> Region.US_EAST_1 + else -> null + } + + this.s3Client = S3AsyncClient.builder() + .credentialsProvider(credentialsProvider) + .apply { + config.endpoint?.let { + endpointOverride(URI.create(it)) + } + + clientRegion?.let { + region(clientRegion) + } + + if (config.virtualHostedStyle != true) { + forcePathStyle(true) + } + } + .build() + } + @Polyglot override fun file(path: String): S3File { + return S3FileProxy(guestExec, s3Client, clientRegion, credentialsProvider, path, config.bucket, config.acl) + } + + override fun getMember(key: String?): Any? = jsObj[key] + override fun getMemberKeys(): Any? = ProxyArray.fromArray(jsObj.keys) + override fun hasMember(key: String?): Boolean = jsObj.containsKey(key) + override fun putMember(key: String?, value: Value?): Nothing = + throw JsError.typeError("Cannot set property '$key' on S3Client object.") +} + +public class S3FileProxy( + private val guestExec: GuestExecutorProvider, + private val s3Client: S3AsyncClient, + private val region: Region?, + private val credentialsProvider: AwsCredentialsProvider, + private val path: String, + private val bucket: String?, + private val acl: String?) : S3File { + + // This map represents the JavaScript object representation of the class to the guest environment + // i.e. it is a map from strings to methods (in this case) + private val jsObj: Map = mapOf( + "write" to ProxyExecutable { args -> + val data = args.getOrNull(0) + ?: throw JsError.typeError("S3File.write() requires one argument for data.") + + val contentType = args.getOrNull(1)?.let { options -> + if (!options.hasMembers()) // options isn't a javascript object + throw JsError.typeError("S3File.write() second argument must be an object containing a 'type' field " + + "signifying MIME type of the data to be written.") + + if (options.hasMember("type")) { + val typeMember = options.getMember("type") + if (typeMember.isString) { + typeMember.asString() + } else { + throw JsError.typeError( + "Option 'type' must be a string, but received a value of type '${typeMember.metaObject.metaSimpleName}'" + ) + } + } else { + null + } + } + + val bufferView = GuestBufferView.tryFrom(data) + when { + data.isString -> write(data.asString(), contentType) + bufferView != null -> write(bufferView.bytes(), contentType) + else -> throw JsError.typeError("Unsupported data type for S3File.write().") + } + }, + "text" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.text() takes zero arguments.") + } + text() + }, + "presign" to ProxyExecutable { args -> + val (method, expiresIn) = args.getOrNull(0)?.let { options -> + if (!options.hasMembers()) { + throw JsError.typeError("S3File.presign() takes an object with optional 'method' and 'expiresIn' properties.") + } + + val method = when { + !options.hasMember("method") -> DEFAULT_PRESIGN_METHOD + options.getMember("method").isString -> options.getMember("method").asString() + else -> throw JsError.typeError("S3File.presign() 'method' property must be a string.") + } + + val expiresIn = when { + !options.hasMember("expiresIn") -> DEFAULT_PRESIGN_DURATION + options.getMember("expiresIn").isNumber -> options.getMember("expiresIn").asLong() + else -> throw JsError.typeError("S3File.presign() 'expiresIn' property must be a number.") + } + + method to expiresIn + } ?: (DEFAULT_PRESIGN_METHOD to DEFAULT_PRESIGN_DURATION) + + presign(method, expiresIn) + }, + "json" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.json() takes zero arguments.") + } + json() + }, + "bytes" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.bytes() takes zero arguments.") + } + bytes() + }, + "arrayBuffer" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.arrayBuffer() takes zero arguments.") + } + arrayBuffer() + }, + "delete" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.delete() takes zero arguments.") + } + delete() + }, + "unlink" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.unlink() takes zero arguments.") + } + delete() + }, + "exists" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.exists() takes zero arguments.") + } + exists() + }, + "stat" to ProxyExecutable { args -> + if (args.isNotEmpty()) { + throw JsError.typeError("S3File.exists() takes stat arguments.") + } + stat() + }, + ) + + private fun write(data: AsyncRequestBody, contentType: String?): JsPromise { + // if content type not specified, infer from path + // default to "application/octet-stream" if content type cannot be inferred from path + val contentType = contentType ?: (URLConnection.guessContentTypeFromName(path) ?: CONTENT_TYPE_DEFAULT) + val request = PutObjectRequest.builder() + .bucket(bucket) + .key(path) + .contentType(contentType) + .apply { + acl?.let { acl(acl) } + } + .build() + val future = guestExec.executor().submit { + s3Client.putObject(request, data).join().size() + } + return JsPromise.wrap(future) + } + + @Polyglot override fun write(data: String, contentType: String?): JsPromise { + return write(AsyncRequestBody.fromString(data), contentType) + } + + @Polyglot override fun write(data: GuestBytes, contentType: String?): JsPromise { + val bytes = ByteArray(data.size) { i -> data[i] } + return write(AsyncRequestBody.fromBytes(bytes), contentType) + } + + @Polyglot override fun write(data: Blob, contentType: String?): JsPromise { + TODO("Blob interface not yet implemented.") + } + + override fun write( + data: ReadableStream, + contentType: String? + ): JsPromise { + TODO("Not yet implemented") + } + + private fun readS3Object(transform: (ResponseBytes) -> T): JsPromise { + val request = GetObjectRequest.builder() + .bucket(bucket) + .key(path) + .build() + val future = guestExec.executor().submit { + val bytes: ResponseBytes = + this.s3Client.getObject(request, AsyncResponseTransformer.toBytes()).get() + transform(bytes) + } + return JsPromise.wrap(future) + } + + @Polyglot override fun text(): JsPromise { + return readS3Object { bytes -> + bytes.asString(StandardCharsets.UTF_8) + } + } + + @Polyglot override fun json(): JsPromise { + return readS3Object { bytes -> + val jsonString = bytes.asString(StandardCharsets.UTF_8) + val context = Context.getCurrent() + context.eval("js", "JSON.parse").execute(jsonString) + } + } + + @Polyglot override fun bytes(): JsPromise { + return readS3Object { bytes -> + val buffer = bytes.asByteBuffer() + ArrayBufferViews.newView(ArrayBufferViewType.Uint8Array, buffer) + } + } + + @Polyglot override fun arrayBuffer(): JsPromise { + return readS3Object { bytes -> + val buffer = bytes.asByteBuffer() + val uint8Array = ArrayBufferViews.newView(ArrayBufferViewType.Uint8Array, buffer) + uint8Array.getMember("buffer") + } + } + + @Polyglot override fun presign(method: String, duration: Long): String { + // There has to be a better way to do this... + val presigner = S3Presigner.builder() + .credentialsProvider(credentialsProvider) + .apply { region?.let { region(region) } } + .build() + val presigned = when (method) { + "PUT" -> { + val putRequest = PutObjectRequest.builder() + .bucket(bucket) + .key(path) + .apply { + acl?.let { acl(acl) } + } + .build() + + val presignRequest = PutObjectPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(duration)) + .putObjectRequest(putRequest) + .build() + + presigner.presignPutObject(presignRequest) + } + "GET" -> { + val getRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(path) + .build() + + val presignRequest = GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(duration)) + .getObjectRequest(getRequest) + .build() + + presigner.presignGetObject(presignRequest) + } + "DELETE" -> { + val deleteRequest = DeleteObjectRequest.builder() + .bucket(bucket) + .key(path) + .build() + val presignRequest = DeleteObjectPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(duration)) + .deleteObjectRequest(deleteRequest) + .build() + presigner.presignDeleteObject(presignRequest) + } + "HEAD" -> { + val headRequest = HeadObjectRequest.builder() + .bucket(bucket) + .key(path) + .build() + val presignRequest = HeadObjectPresignRequest.builder() + .signatureDuration(Duration.ofSeconds(duration)) + .headObjectRequest(headRequest) + .build() + presigner.presignHeadObject(presignRequest) + } + else -> throw JsError.typeError("S3File.presign() only takes GET, PUT, DELETE, and HEAD methods.") + } + return presigned.url().toExternalForm() + } + + @Polyglot override fun delete(): JsPromise { + val request = DeleteObjectRequest.builder() + .key(path) + .bucket(bucket) + .build() + val future = guestExec.executor().submit { + s3Client.deleteObject(request).join() + } + return JsPromise.wrap(future) + } + + @Polyglot override fun exists(): JsPromise { + val request = HeadObjectRequest.builder() + .key(path) + .bucket(bucket) + .build() + val future = guestExec.executor().submit { + try { + s3Client.headObject(request).join() + true + } catch (e: CompletionException) { + if (e.cause is NoSuchKeyException) false + else throw e + } + } + return JsPromise.wrap(future) + } + + @Polyglot override fun stat(): JsPromise { + val request = HeadObjectRequest.builder() + .key(path) + .bucket(bucket) + .build() + val future = guestExec.executor().submit { + val response = s3Client.headObject(request).join() + val statData = mapOf( + "eTag" to response.eTag(), + "contentType" to response.contentType(), + "contentLength" to response.contentLength(), + "lastModified" to response.lastModified().toString(), + "metadata" to ProxyObject.fromMap(response.metadata() as Map) + ) + ProxyObject.fromMap(statData) + } + return JsPromise.wrap(future) + } + + override fun getMember(key: String?): Any? = jsObj[key] + override fun getMemberKeys(): Any? = ProxyArray.fromArray(jsObj.keys) + override fun hasMember(key: String?): Boolean = jsObj.containsKey(key) + override fun putMember(key: String?, value: Value?): Nothing = + throw JsError.typeError("Cannot set property '$key' on S3File object.") +} diff --git a/packages/graalvm/src/test/kotlin/elide/runtime/intrinsics/s3/S3ModuleTest.kt b/packages/graalvm/src/test/kotlin/elide/runtime/intrinsics/s3/S3ModuleTest.kt new file mode 100644 index 0000000000..e2218f31e3 --- /dev/null +++ b/packages/graalvm/src/test/kotlin/elide/runtime/intrinsics/s3/S3ModuleTest.kt @@ -0,0 +1,580 @@ +package elide.runtime.intrinsics.s3 + +import com.robothy.s3.rest.LocalS3 +import com.robothy.s3.rest.bootstrap.LocalS3Mode +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.assertNotNull +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.CreateBucketRequest +import software.amazon.awssdk.services.s3.model.GetObjectRequest +import software.amazon.awssdk.services.s3.model.NoSuchKeyException +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import java.net.URI +import java.nio.charset.StandardCharsets +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.fail +import elide.annotations.Inject +import elide.runtime.gvm.internals.js.AbstractJsIntrinsicTest +import elide.runtime.intrinsics.js.s3.S3Module +import elide.testing.annotations.Test +import elide.testing.annotations.TestCase + +@TestCase internal class S3ModuleTest : AbstractJsIntrinsicTest() { + @Inject lateinit var module: S3Module + override fun provide(): S3Module = module + + companion object { + private lateinit var s3Server: LocalS3 + + // Generates an S3 client for creating test scenarios. + @JvmStatic + fun s3Client() = S3Client.builder() + .forcePathStyle(true) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .endpointOverride(URI.create(testServerEndpoint())) + .region(Region.US_WEST_2) + .build() + + // Generates the endpoint string for testing. Do not run before the server has started. + @JvmStatic + fun testServerEndpoint() = "http://localhost:${s3Server.port}" + + @JvmStatic + fun clientPrelude() = """ + const { S3Client } = require("elide:s3"); + const client = new S3Client({ + endpoint: "${testServerEndpoint()}", + region: "us-west-2", + accessKeyId: "test", + secretAccessKey: "test", + bucket: "test" + }); + """ + + @JvmStatic + @BeforeAll + fun startup() { + s3Server = LocalS3.builder() + .port(-1) + .mode(LocalS3Mode.IN_MEMORY) + .build() + s3Server.start() + + // create test bucket + val request = CreateBucketRequest.builder() + .bucket("test") + .build() + + s3Client().createBucket(request) + } + + @JvmStatic + @AfterAll + fun shutdown() { + s3Server.shutdown() + } + } + + @Test override fun testInjectable() { + assertNotNull(module) { "S3 module should be injectable" } + } + + @Test fun testClientConstructor() { + // required options + executeGuest { + // language=javascript + """ + const { S3Client } = require("elide:s3"); + const client = new S3Client({}); + """ + }.fails() + + // not enough args + executeGuest { + // language=javascript + """ + const { S3Client } = require("elide:s3"); + const client = new S3Client(); + """ + }.fails() + + // args wrong type + executeGuest { + // language=javascript + """ + const { S3Client } = require("elide:s3"); + const client = new S3Client("test"); + """ + }.fails() + } + + @Test fun testFileConstructor() { + // zero args + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file(); + """ + }.fails() + + // wrong type + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file({}); + """ + }.fails() + } + + @Test fun testWriteString() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("write-test.txt"); + file.write("test").then(_ => {}); + """ + }.doesNotFail() + val request = GetObjectRequest.builder() + .key("write-test.txt") + .bucket("test") + .build() + val response = s3Client().getObject(request) + assertEquals("test", response.readAllBytes().toString(StandardCharsets.UTF_8)) + assertEquals("text/plain", response.response().contentType()) + } + + @Test fun testDefaultContentType() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("default-content-type-test"); + file.write("test").then(_ => {}); + """ + }.doesNotFail() + val request = GetObjectRequest.builder() + .key("default-content-type-test") + .bucket("test") + .build() + val response = s3Client().getObject(request) + assertEquals("test", response.readAllBytes().toString(StandardCharsets.UTF_8)) + assertEquals("application/octet-stream", response.response().contentType()) + } + + @Test fun testContentType() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("content-type-test"); + file.write("test", { type: "fake/this" }).then(_ => {}); + """ + }.doesNotFail() + val request = GetObjectRequest.builder() + .key("content-type-test") + .bucket("test") + .build() + val response = s3Client().getObject(request) + assertEquals("test", response.readAllBytes().toString(StandardCharsets.UTF_8)) + assertEquals("fake/this", response.response().contentType()) + } + + @Test fun testContentTypeFailures() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("content-type-test"); + file.write("test", "plain/text").then(_ => {}); + """ + }.fails() + + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("content-type-test"); + file.write("test", { type: {} }).then(_ => {}); + """ + }.fails() + } + + @Test fun testWriteArgsFailures() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("write-fail-test.txt"); + file.write(); + """ + }.fails() + + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("write-fail-test.txt"); + file.write({ hello: "world" }); + """ + }.fails() + } + + @Test fun testWriteArray() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("write-array-test.txt"); + file.write(new Uint8Array([1, 2, 3])).then(_ => {}); + """ + }.doesNotFail() + val request = GetObjectRequest.builder() + .key("write-array-test.txt") + .bucket("test") + .build() + val response = s3Client().getObject(request) + assertContentEquals(byteArrayOf(1, 2, 3), response.readAllBytes()) + } + + @Test fun testText() { + val request = PutObjectRequest.builder() + .key("text-test.txt") + .bucket("test") + .build() + s3Client().putObject(request, RequestBody.fromString("text test")) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("text-test.txt"); + file.text().then(out => { + test(out).isNotNull(); + test(out).isEqualTo("text test"); + }); + """ + }.doesNotFail() + } + + @Test fun testJson() { + val request = PutObjectRequest.builder() + .key("json-test.json") + .bucket("test") + .build() + s3Client().putObject(request, RequestBody.fromString("""{"test": "value"}""")) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("json-test.json"); + file.json().then(out => { + test(out).isNotNull(); + test(out.test).isNotNull(); + test(out.test).isEqualTo("value"); + }); + """ + }.doesNotFail() + } + + @Test fun testRoundtripString() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("test.txt"); + file.write("test").then(_ => { + file.text().then(output => { + test(output).isNotNull(); + test(output).isEqualTo("test"); + }); + }); + """ + }.doesNotFail() + } + + @Test fun testRoundtripJson() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("test.json"); + const data = { "test": "value" }; + file.write(JSON.stringify(data)).then(_ => { + file.json().then(output => { + test(output).isNotNull(); + test(output.test).isNotNull(); + test(output.test).isEqualTo("value"); + }); + }); + """ + }.doesNotFail() + } + + @Test fun testBytes() { + val request = PutObjectRequest.builder() + .key("bytes-test.bin") + .bucket("test") + .build() + val content = byteArrayOf(1, 2, 3, 4, 5) + s3Client().putObject(request, RequestBody.fromBytes(content)) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("bytes-test.bin"); + file.bytes().then(out => { + test(out).isNotNull(); + test(out instanceof Uint8Array).isEqualTo(true); + const expected = new Uint8Array([1, 2, 3, 4, 5]); + test(out.length).isEqualTo(expected.length); + test(out.every((v, i) => v === expected[i])).isEqualTo(true); + }); + """ + }.doesNotFail() + } + + @Test fun testRoundtripBytes() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("roundtrip-bytes.bin"); + const data = new Uint8Array([5, 4, 3, 2, 1]); + file.write(data).then(_ => { + file.bytes().then(out => { + test(out).isNotNull(); + test(out instanceof Uint8Array).isEqualTo(true); + test(out.length).isEqualTo(data.length); + test(out.every((v, i) => v === data[i])).isEqualTo(true); + }); + }); + """ + }.doesNotFail() + } + + @Test fun testArrayBuffer() { + val request = PutObjectRequest.builder() + .key("arraybuffer-test.bin") + .bucket("test") + .build() + val content = byteArrayOf(1, 2, 3, 4, 5) + s3Client().putObject(request, RequestBody.fromBytes(content)) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("arraybuffer-test.bin"); + file.arrayBuffer().then(out => { + test(out).isNotNull(); + test(out instanceof ArrayBuffer).isEqualTo(true); + const expected = [1, 2, 3, 4, 5] + const view = new Uint8Array(out); + test(view.length).isEqualTo(expected.length); + test(view.every((v, i) => v === expected[i])).isEqualTo(true); + }); + """ + }.doesNotFail() + } + + @Test fun testRoundtripArrayBuffer() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("roundtrip-arraybuffer.bin"); + const data = new Uint8Array([5, 4, 3, 2, 1]); + file.write(data).then(_ => { + file.arrayBuffer().then(output => { + test(output).isNotNull(); + test(output instanceof ArrayBuffer).isEqualTo(true); + const view = new Uint8Array(output); + test(view.length).isEqualTo(data.length); + test(view.every((v, i) => v === data[i])).isEqualTo(true); + }); + }); + """ + }.doesNotFail() + } + + @Test fun testPresign() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-test.txt"); + const url = file.presign(); + test(url).isNotNull(); + test(url.includes("presign-test.txt")).isEqualTo(true); + """ + }.doesNotFail() + } + + @Test fun testPresignWithMethod() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-method-test.txt"); + const url = file.presign({ method: "PUT" }); + test(url).isNotNull(); + test(url.includes("presign-method-test.txt")).isEqualTo(true); + """ + }.doesNotFail() + } + + @Test fun testPresignWithExpiresIn() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-expires-test.txt"); + const url = file.presign({ expiresIn: 100 }); + test(url).isNotNull(); + test(url.includes("presign-expires-test.txt")).isEqualTo(true); + """ + }.doesNotFail() + } + + @Test fun testPresignWithMethodAndExpiresIn() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-full-test.txt"); + const url = file.presign({ method: "DELETE", expiresIn: 200 }); + test(url).isNotNull(); + test(url.includes("presign-full-test.txt")).isEqualTo(true); + """ + }.doesNotFail() + } + + @Test fun testPresignFailures() { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-fail-test.txt"); + file.presign({ method: 123 }); + """ + }.fails() + + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-fail-test.txt"); + file.presign({ expiresIn: "123" }); + """ + }.fails() + + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("presign-fail-test.txt"); + file.presign({ method: "INVALID" }); + """ + }.fails() + } + + @ParameterizedTest + @ValueSource(strings = ["delete", "unlink"]) + fun testDelete(funcName: String) { + val putRequest = PutObjectRequest.builder() + .key("${funcName}-test.txt") + .bucket("test") + .build() + s3Client().putObject(putRequest, RequestBody.fromString("${funcName} test")) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("${funcName}-test.txt"); + file.${funcName}(); + """ + }.doesNotFail() + val request = GetObjectRequest.builder() + .key("${funcName}-test.txt") + .bucket("test") + .build() + try { + s3Client().getObject(request) + } catch (_: NoSuchKeyException) { + return + } + fail("Object was not deleted.") + } + + @ParameterizedTest + @ValueSource(strings = ["delete", "unlink", "exists", "stat", "arrayBuffer", + "bytes", "json", "text"]) + fun testNoArgs(funcName: String) { + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("${funcName}-fail-test.txt"); + file.${funcName}("some argument"); + """ + }.fails() + } + + @Test fun testExists() { + val request = PutObjectRequest.builder() + .key("exists-test.bin") + .bucket("test") + .build() + val content = byteArrayOf(1, 2, 3, 4, 5) + s3Client().putObject(request, RequestBody.fromBytes(content)) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("exists-test.bin"); + file.exists().then(out => { + test(out).isEqualTo(true); + }); + const notAFile = client.file("exists-test-does-not-exist.bin"); + notAFile.exists().then(out => { + test(out).isEqualTo(false); + }); + """ + }.doesNotFail() + } + + @Test fun testStat() { + val metadata = mapOf("test-key" to "test-value") + val putRequest = PutObjectRequest.builder() + .key("stat-test.txt") + .bucket("test") + .metadata(metadata) + .contentType("text/plain") + .build() + s3Client().putObject(putRequest, RequestBody.fromString("stat test")) + executeGuest { + // language=javascript + """ + ${clientPrelude()} + const file = client.file("stat-test.txt"); + file.stat().then(out => { + test(out).isNotNull(); + test(out.eTag).isNotNull(); + test(out.contentType).isEqualTo("text/plain"); + test(out.contentLength === 9).isEqualTo(true); + test(out.lastModified).isNotNull(); + test(out.metadata).isNotNull(); + test(out.metadata["test-key"]).isEqualTo("test-value"); + }); + """ + }.doesNotFail() + } +}