Skip to content

Commit 2ef2d5b

Browse files
JetpackDubaJetpackDuba
committed
Improved LFS implementation
- Added support for dynamic headers that support alternative methods to basic auth. - Added LFS uploads verification. - Added support for implicit LFS url
1 parent 794f90c commit 2ef2d5b

File tree

5 files changed

+122
-29
lines changed

5 files changed

+122
-29
lines changed

‎src/main/kotlin/com/jetpackduba/gitnuro/lfs/GLfsFactory.kt

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,8 @@ class GLfsPrePushHook(
237237
}
238238

239239
if (!isDryRun) {
240-
val lfsServerUrl = lfsRepository.getLfsRepositoryUrl(repository)
240+
val lfsServerUrl =
241+
lfsRepository.getLfsRepositoryUrl(repository, remote()) ?: throw Exception("LFS Url not found")
241242

242243
val lfsPrepareUploadObjectBatch = createLfsPrepareUploadObjectBatch(
243244
OperationType.UPLOAD,
@@ -249,7 +250,8 @@ class GLfsPrePushHook(
249250
var credentialsAlreadyRequested = false
250251

251252
val cachedCredentials = run {
252-
val cacheHttpCredentials = credentialsCacheRepository.getCachedHttpCredentials(lfsServerUrl, isLfs = true)
253+
val cacheHttpCredentials =
254+
credentialsCacheRepository.getCachedHttpCredentials(lfsServerUrl, isLfs = true)
253255

254256
if (cacheHttpCredentials != null) {
255257
CredentialsAccepted.LfsCredentialsAccepted.fromCachedCredentials(cacheHttpCredentials)
@@ -300,13 +302,20 @@ class GLfsPrePushHook(
300302

301303
if (uploadUrl != null) {
302304
lfsRepository.uploadObject(
305+
obj,
303306
uploadUrl,
304307
p.oid.name(),
305308
lfs.getMediaFile(p.oid),
306309
p.size,
307310
credentials?.user,
308311
credentials?.password,
309312
)
313+
314+
lfsRepository.verify(
315+
obj,
316+
credentials?.user,
317+
credentials?.password,
318+
)
310319
}
311320
}
312321
}

‎src/main/kotlin/com/jetpackduba/gitnuro/lfs/LfsNetworkDataSource.kt

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.jetpackduba.gitnuro.lfs
22

33
import com.jetpackduba.gitnuro.Result
4+
import com.jetpackduba.gitnuro.models.lfs.LfsObject
45
import com.jetpackduba.gitnuro.models.lfs.LfsObjects
56
import com.jetpackduba.gitnuro.models.lfs.LfsPrepareUploadObjectBatch
67
import io.ktor.client.*
@@ -34,6 +35,7 @@ interface ILfsNetworkDataSource {
3435
)
3536

3637
suspend fun uploadObject(
38+
lfsObject: LfsObject,
3739
uploadUrl: String,
3840
oid: String,
3941
file: Path,
@@ -42,7 +44,14 @@ interface ILfsNetworkDataSource {
4244
password: String?,
4345
)
4446

47+
suspend fun verify(
48+
lfsObject: LfsObject,
49+
username: String?,
50+
password: String?,
51+
)
52+
4553
suspend fun downloadObject(
54+
lfsObject: LfsObject,
4655
downloadUrl: String,
4756
outPath: Path,
4857
username: String?,
@@ -53,6 +62,8 @@ interface ILfsNetworkDataSource {
5362
class LfsNetworkDataSource @Inject constructor(
5463
private val client: HttpClient,
5564
) : ILfsNetworkDataSource {
65+
private val json = Json { ignoreUnknownKeys = true }
66+
5667
override suspend fun postBatchObjects(
5768
remoteUrl: String,
5869
lfsPrepareUploadObjectBatch: LfsPrepareUploadObjectBatch,
@@ -69,15 +80,14 @@ class LfsNetworkDataSource @Inject constructor(
6980
basicAuth(username, password)
7081
}
7182

72-
setBody(Json.encodeToString(lfsPrepareUploadObjectBatch))
83+
setBody(json.encodeToString(lfsPrepareUploadObjectBatch))
7384
}
7485

7586
if (response.status != HttpStatusCode.OK) {
7687
return Result.Err(LfsError.HttpError(response.status))
7788
}
7889

79-
80-
return Result.Ok(Json.decodeFromString(response.bodyAsText()))
90+
return Result.Ok(json.decodeFromString(response.bodyAsText()))
8191
}
8292

8393
override suspend fun uploadBatchObjects(
@@ -97,7 +107,7 @@ class LfsNetworkDataSource @Inject constructor(
97107
}
98108

99109

100-
this.setBody(Json.encodeToString(lfsPrepareUploadObjectBatch))
110+
this.setBody(json.encodeToString(lfsPrepareUploadObjectBatch))
101111
}
102112

103113
if (response.status.value != 200) {
@@ -106,6 +116,7 @@ class LfsNetworkDataSource @Inject constructor(
106116
}
107117

108118
override suspend fun uploadObject(
119+
lfsObject: LfsObject,
109120
uploadUrl: String,
110121
oid: String,
111122
file: Path,
@@ -115,7 +126,23 @@ class LfsNetworkDataSource @Inject constructor(
115126
) {
116127
val response = client.put(uploadUrl) {
117128
if (username != null && password != null) {
118-
basicAuth(username, password)
129+
val objHeaders = lfsObject.actions.upload?.header
130+
131+
if (objHeaders.isNullOrEmpty()) {
132+
basicAuth(username, password)
133+
} else {
134+
for (header in objHeaders.entries) {
135+
header(header.key, header.value)
136+
}
137+
}
138+
139+
140+
// if (authorization != null) {
141+
//
142+
// bearerAuth(authorization)
143+
// } else {
144+
// basicAuth(username, password)
145+
// }
119146
}
120147

121148
this.headers {
@@ -131,7 +158,32 @@ class LfsNetworkDataSource @Inject constructor(
131158
}
132159
}
133160

161+
override suspend fun verify(lfsObject: LfsObject, username: String?, password: String?) {
162+
val response = client.post(lfsObject.actions.verify!!.href) {
163+
if (username != null && password != null) {
164+
val objHeaders = lfsObject.actions.verify?.header
165+
166+
if (objHeaders.isNullOrEmpty()) {
167+
basicAuth(username, password)
168+
} else {
169+
for (header in objHeaders.entries) {
170+
header(header.key, header.value)
171+
}
172+
}
173+
}
174+
175+
this.headers {
176+
this["Accept"] = "application/vnd.git-lfs"
177+
}
178+
}
179+
180+
if (response.status != HttpStatusCode.OK) {
181+
throw Exception("Verify status is ${response.status} instead of HTTP OK 200")
182+
}
183+
}
184+
134185
override suspend fun downloadObject(
186+
lfsObject: LfsObject,
135187
downloadUrl: String,
136188
outPath: Path,
137189
username: String?,

‎src/main/kotlin/com/jetpackduba/gitnuro/lfs/LfsRepository.kt

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,63 @@
11
package com.jetpackduba.gitnuro.lfs
22

3+
import com.jetpackduba.gitnuro.git.branches.GetBranchesUseCase
4+
import com.jetpackduba.gitnuro.git.branches.GetCurrentBranchUseCase
5+
import com.jetpackduba.gitnuro.git.branches.GetTrackingBranchUseCase
6+
import com.jetpackduba.gitnuro.logging.printError
37
import com.jetpackduba.gitnuro.models.lfs.LfsObjectBatch
48
import com.jetpackduba.gitnuro.models.lfs.LfsPrepareUploadObjectBatch
59
import com.jetpackduba.gitnuro.models.lfs.LfsRef
10+
import org.eclipse.jgit.api.Git
11+
import org.eclipse.jgit.lib.Constants
612
import org.eclipse.jgit.lib.Repository
713
import org.eclipse.jgit.storage.file.FileBasedConfig
814
import java.io.File
915
import javax.inject.Inject
1016

17+
private const val TAG = "LfsRepository"
18+
1119
class LfsRepository @Inject constructor(
1220
private val lfsNetworkDataSource: LfsNetworkDataSource,
1321
) : ILfsNetworkDataSource by lfsNetworkDataSource {
14-
fun getLfsRepositoryUrl(repository: Repository): String {
22+
suspend fun getLfsRepositoryUrl(repository: Repository, remoteName: String?): String? {
1523
// TODO This only gets the URL from considering happy path config
1624
val configFile = File(repository.workTree, ".lfsconfig")
1725

1826
val config = FileBasedConfig(configFile, repository.fs)
1927
config.load()
2028

21-
return config.getString("lfs", null, "url")
29+
val lfsConfigUrl = config.getString("lfs", null, "url")
30+
31+
return lfsConfigUrl ?: getLfsUrlFromRemote(repository, remoteName)
32+
}
33+
34+
private suspend fun getLfsUrlFromRemote(repository: Repository, remoteName: String?): String? {
35+
val remotePath = if(remoteName != null) {
36+
remoteName
37+
} else {
38+
val q = GetTrackingBranchUseCase().invoke(Git(repository), GetCurrentBranchUseCase(GetBranchesUseCase()).invoke(Git(repository))!!)
39+
val currentBranchTracedRemote = q?.remote
40+
41+
if (currentBranchTracedRemote != null) {
42+
Constants.R_REMOTES + currentBranchTracedRemote
43+
} else {
44+
printError(TAG, "Remote name is null and couldn't obtain tracking branch remote.")
45+
return null
46+
}
47+
}
48+
49+
val remoteUrl = Git(repository)
50+
.remoteList()
51+
.call()
52+
.firstOrNull { Constants.R_REMOTES + it.name == remotePath }
53+
?.urIs
54+
?.firstOrNull()
55+
56+
return if (remoteUrl != null) {
57+
"$remoteUrl/info/lfs"
58+
} else {
59+
null
60+
}
2261
}
2362
}
2463

‎src/main/kotlin/com/jetpackduba/gitnuro/lfs/LfsSmudgeFilter.kt

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package com.jetpackduba.gitnuro.lfs
33
import com.jetpackduba.gitnuro.Result
44
import com.jetpackduba.gitnuro.credentials.CredentialsAccepted
55
import com.jetpackduba.gitnuro.credentials.CredentialsCacheRepository
6-
import com.jetpackduba.gitnuro.credentials.CredentialsRequest
76
import com.jetpackduba.gitnuro.credentials.CredentialsStateManager
87
import com.jetpackduba.gitnuro.logging.printLog
98
import com.jetpackduba.gitnuro.models.lfs.LfsObjectBatch
@@ -17,9 +16,7 @@ import org.eclipse.jgit.lfs.LfsPointer
1716
import org.eclipse.jgit.lfs.lib.AnyLongObjectId
1817
import org.eclipse.jgit.lib.Repository
1918
import java.io.*
20-
import java.net.URI
2119
import java.nio.file.Files
22-
import java.util.concurrent.CancellationException
2320

2421
private const val MAX_COPY_BYTES = 1024 * 1024 * 256
2522

@@ -59,7 +56,7 @@ class LfsSmudgeFilter(
5956
repository: Repository,
6057
res: LfsPointer,
6158
) = runBlocking {
62-
val lfsServerUrl = lfsRepository.getLfsRepositoryUrl(repository)
59+
val lfsServerUrl = lfsRepository.getLfsRepositoryUrl(repository, null) ?: throw Exception("LFS Url not found")
6360

6461
val hash = res.oid.name()
6562
val size = res.size
@@ -146,6 +143,7 @@ class LfsSmudgeFilter(
146143

147144
do {
148145
val newLfsObjects = lfsRepository.downloadObject(
146+
lfsObject,
149147
downloadUrl,
150148
lfs.getMediaFile(oid),
151149
username,

‎src/main/kotlin/com/jetpackduba/gitnuro/models/lfs/LfsObjects.kt

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,20 @@ data class LfsObject(
1818

1919
@Serializable
2020
data class Actions(
21-
@SerialName("download") var download: Download? = null,
22-
@SerialName("upload") var upload: Upload? = null,
21+
@SerialName("download") var download: RemoteObjectAccessInfo? = null,
22+
@SerialName("upload") var upload: RemoteObjectAccessInfo? = null,
23+
@SerialName("verify") var verify: RemoteObjectAccessInfo? = null,
2324
)
2425

2526
@Serializable
26-
data class Download(
27-
@SerialName("href") var href: String? = null,
28-
@SerialName("header") var header: Header? = Header(),
29-
@SerialName("expires_at") var expiresAt: String? = null
30-
)
31-
32-
@Serializable
33-
data class Upload(
27+
data class RemoteObjectAccessInfo(
3428
@SerialName("href") var href: String,
35-
@SerialName("header") var header: Header?,
36-
@SerialName("expires_at") var expiresAt: String?,
29+
@SerialName("header") var header: Map<String, String>? = null,
30+
@SerialName("expires_at") var expiresAt: String? = null,
3731
)
3832

39-
@Serializable
40-
data class Header(
41-
@SerialName("Accept") var accept: String? = null
42-
)
33+
//@Serializable
34+
//data class Header(
35+
// @SerialName("Accept") var accept: String? = null,
36+
// @SerialName("Authorization") var authorization: String? = null,
37+
//)

0 commit comments

Comments
 (0)