Skip to content

Commit 66e3b1d

Browse files
BarDwellerBarDweller
committed
Add docker options for pull retry
1 parent fb6f150 commit 66e3b1d

File tree

11 files changed

+186
-70
lines changed

11 files changed

+186
-70
lines changed

‎README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ The [`BuildpackConfig`](client/src/main/java/dev/snowdrop/buildpack/BuildConfig.
3333

3434
- run/build/output Image can be specified
3535
- docker can be configured with..
36-
- pull timeout
36+
- pull timeout
37+
- pull retry count (will retry image pull on failure)
38+
- pull retry timeout increase (increases timeout each time pull is retried)
3739
- host
3840
- network
3941
- docker socket path

‎client/src/main/java/dev/snowdrop/buildpack/BuilderImage.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ public BuilderImage(DockerConfig dc, PlatformConfig pc, ImageReference runImage,
4444
image = builderImage;
4545

4646
// pull and inspect the builderImage to obtain builder metadata.
47-
ImageUtils.pullImages(dc.getDockerClient(),
48-
dc.getPullTimeout(),
49-
builderImage.getReference());
47+
ImageUtils.pullImages(dc, builderImage.getReference());
5048

5149
ImageInfo ii = ImageUtils.inspectImage(dc.getDockerClient(),
5250
builderImage.getReference());

‎client/src/main/java/dev/snowdrop/buildpack/config/DockerConfig.java

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,17 @@ public static DockerConfigBuilder builder() {
1212
return new DockerConfigBuilder();
1313
}
1414

15+
public static enum PullPolicy {ALWAYS, IF_NOT_PRESENT};
16+
1517
private static final Integer DEFAULT_PULL_TIMEOUT = 60;
18+
private static final Integer DEFAULT_PULL_RETRY_INCREASE = 15;
19+
private static final Integer DEFAULT_PULL_RETRY_COUNT = 3;
20+
private static final PullPolicy DEFAULT_PULL_POLICY = PullPolicy.IF_NOT_PRESENT;
1621

1722
private Integer pullTimeoutSeconds;
23+
private Integer pullRetryCount;
24+
private Integer pullRetryIncreaseSeconds;
25+
private PullPolicy pullPolicy;
1826
private String dockerHost;
1927
private String dockerSocket;
2028
private String dockerNetwork;
@@ -23,13 +31,19 @@ public static DockerConfigBuilder builder() {
2331

2432
public DockerConfig(
2533
Integer pullTimeoutSeconds,
34+
Integer pullRetryCount,
35+
Integer pullRetryIncreaseSeconds,
36+
PullPolicy pullPolicy,
2637
String dockerHost,
2738
String dockerSocket,
2839
String dockerNetwork,
2940
Boolean useDaemon,
3041
DockerClient dockerClient
3142
){
32-
this.pullTimeoutSeconds = pullTimeoutSeconds != null ? pullTimeoutSeconds : DEFAULT_PULL_TIMEOUT;
43+
this.pullTimeoutSeconds = pullTimeoutSeconds != null ? Integer.max(0,pullTimeoutSeconds) : DEFAULT_PULL_TIMEOUT;
44+
this.pullRetryCount = pullRetryCount != null ? Integer.max(0,pullRetryCount) : DEFAULT_PULL_RETRY_COUNT;
45+
this.pullRetryIncreaseSeconds = pullRetryIncreaseSeconds != null ? Integer.max(0,pullRetryIncreaseSeconds) : DEFAULT_PULL_RETRY_INCREASE;
46+
this.pullPolicy = pullPolicy != null ? pullPolicy : DEFAULT_PULL_POLICY;
3347
this.dockerHost = dockerHost != null ? dockerHost : DockerClientUtils.getDockerHost();
3448
this.dockerSocket = dockerSocket != null ? dockerSocket : (this.dockerHost.startsWith("unix://") ? this.dockerHost.substring("unix://".length()) : "/var/run/docker.sock");
3549
this.dockerNetwork = dockerNetwork;
@@ -47,6 +61,18 @@ public Integer getPullTimeout(){
4761
return this.pullTimeoutSeconds;
4862
}
4963

64+
public Integer getPullRetryCount(){
65+
return this.pullRetryCount;
66+
}
67+
68+
public Integer getPullRetryIncrease(){
69+
return this.pullRetryIncreaseSeconds;
70+
}
71+
72+
public PullPolicy getPullPolicy(){
73+
return this.pullPolicy;
74+
}
75+
5076
public String getDockerHost(){
5177
return this.dockerHost;
5278
}

‎client/src/main/java/dev/snowdrop/buildpack/docker/ImageUtils.java

Lines changed: 74 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@
22

33
import java.util.ArrayList;
44
import java.util.Arrays;
5+
import java.util.HashMap;
56
import java.util.HashSet;
67
import java.util.List;
78
import java.util.Map;
9+
import java.util.Map.Entry;
810
import java.util.Set;
911
import java.util.concurrent.TimeUnit;
12+
import java.util.stream.Collectors;
1013

1114
import org.slf4j.Logger;
1215
import org.slf4j.LoggerFactory;
@@ -15,7 +18,10 @@
1518
import com.github.dockerjava.api.command.InspectImageResponse;
1619
import com.github.dockerjava.api.command.PullImageResultCallback;
1720
import com.github.dockerjava.api.model.Image;
18-
21+
import com.github.dockerjava.api.exception.DockerClientException;
22+
import com.github.dockerjava.api.exception.NotFoundException;
23+
24+
import dev.snowdrop.buildpack.config.DockerConfig;
1925
import dev.snowdrop.buildpack.BuildpackException;
2026
/**
2127
* Higher level docker image api
@@ -30,51 +36,93 @@ public static class ImageInfo {
3036
}
3137

3238
/**
33-
* Util method to pull images if they don't exist to the local docker yet.
39+
* Util method to pull images, configure behavior via dockerconfig.
3440
*/
35-
public static void pullImages(DockerClient dc, int timeoutSeconds, String... imageNames) {
41+
@SuppressWarnings("resource")
42+
public static void pullImages(DockerConfig config, String... imageNames) {
3643
Set<String> imageNameSet = new HashSet<>(Arrays.asList(imageNames));
3744

38-
// list the current known images
39-
List<Image> li = dc.listImagesCmd().exec();
40-
for (Image i : li) {
41-
if (i.getRepoTags() == null) {
42-
continue;
43-
}
44-
for (String it : i.getRepoTags()) {
45-
if (imageNameSet.contains(it)) {
46-
imageNameSet.remove(it);
45+
DockerClient dc = config.getDockerClient();
46+
47+
//if using ifnotpresent, filter set to unknown images.
48+
if(config.getPullPolicy() == DockerConfig.PullPolicy.IF_NOT_PRESENT) {
49+
// list the current known images
50+
List<Image> li = dc.listImagesCmd().exec();
51+
for (Image i : li) {
52+
if (i.getRepoTags() == null) {
53+
continue;
54+
}
55+
for (String it : i.getRepoTags()) {
56+
if (imageNameSet.contains(it)) {
57+
imageNameSet.remove(it);
58+
}
4759
}
4860
}
49-
}
5061

51-
if (imageNameSet.isEmpty()) {
52-
// fast exit if all images are already known to the local docker.
53-
log.debug("Nothing to pull, all of " + Arrays.asList(imageNames) + " are known");
54-
return;
62+
if (imageNameSet.isEmpty()) {
63+
// fast exit if all images are already known to the local docker.
64+
log.debug("Nothing to pull, all of " + Arrays.asList(imageNames) + " are known");
65+
return;
66+
}
5567
}
5668

57-
// pull the images not known
58-
List<PullImageResultCallback> pircs = new ArrayList<>();
69+
int retryCount = 0;
70+
Map<String,PullImageResultCallback> pircMap = new HashMap<>();
71+
72+
// pull the images still in set.
5973
for (String stillNeeded : imageNameSet) {
6074
log.debug("pulling '" + stillNeeded + "'");
6175
PullImageResultCallback pirc = new PullImageResultCallback();
6276
dc.pullImageCmd(stillNeeded).exec(pirc);
63-
pircs.add(pirc);
77+
pircMap.put(stillNeeded,pirc);
6478
}
6579

6680
// wait for pulls to complete.
67-
for (PullImageResultCallback pirc : pircs) {
68-
try {
69-
pirc.awaitCompletion(timeoutSeconds, TimeUnit.SECONDS);
70-
} catch (InterruptedException e) {
71-
throw BuildpackException.launderThrowable(e);
81+
RuntimeException lastSeen = null;
82+
boolean allDone = false;
83+
while(!allDone && retryCount<=config.getPullRetryCount()){
84+
allDone = true;
85+
long thisWait = config.getPullTimeout()+(retryCount*config.getPullRetryIncrease());
86+
for (Entry<String, PullImageResultCallback> e : pircMap.entrySet()) {
87+
boolean done = false;
88+
try {
89+
if(e.getValue()==null) continue;
90+
log.debug("waiting on image "+e.getKey()+" for "+thisWait+" seconds");
91+
done = e.getValue().awaitCompletion( thisWait, TimeUnit.SECONDS);
92+
log.debug("success for image "+e.getKey());
93+
} catch (InterruptedException ie) {
94+
throw BuildpackException.launderThrowable(ie);
95+
} catch (DockerClientException dce) {
96+
//error occurred during pull for this pirc, need to pause & retry the pull op
97+
lastSeen = dce;
98+
} catch (NotFoundException nfe) {
99+
lastSeen = nfe;
100+
}
101+
if(!done){
102+
String imageName = e.getKey();
103+
PullImageResultCallback newPirc = new PullImageResultCallback();
104+
dc.pullImageCmd(imageName).exec(newPirc);
105+
e.setValue(newPirc);
106+
allDone=false;
107+
}else{
108+
e.setValue(null);
109+
}
110+
}
111+
retryCount++;
112+
if(retryCount<=config.getPullRetryCount()){
113+
if(lastSeen!=null){
114+
log.debug("Error during pull "+lastSeen.getMessage());
115+
}
116+
log.debug("Retrying ("+retryCount+") for "+pircMap.entrySet().stream().filter(e -> e.getValue()!=null).collect(Collectors.toList()));
72117
}
73118
}
74119

75-
// TODO: progress tracking..
120+
if(lastSeen!=null && !allDone){
121+
throw lastSeen;
122+
}
76123
}
77124

125+
78126
/**
79127
* Util method to retrieve info for a given docker image.
80128
*/

‎client/src/main/java/dev/snowdrop/buildpack/lifecycle/LifecycleExecutor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public int execute() {
105105
}
106106

107107
//pull the new image..
108-
ImageUtils.pullImages(config.getDockerConfig().getDockerClient(), factory.getDockerConfig().getPullTimeout(), newRunImage);
108+
ImageUtils.pullImages(config.getDockerConfig(), newRunImage);
109109

110110
//update run image associated with our builder image.
111111
factory.getBuilderImage().getRunImages(activePlatformLevel).clear();

‎client/src/main/java/dev/snowdrop/buildpack/utils/LifecycleMetadata.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,7 @@ public class LifecycleMetadata {
2020
public LifecycleMetadata(DockerConfig dc, ImageReference lifecycleImage) throws BuildpackException {
2121

2222
// pull and inspect the builderImage to obtain builder metadata.
23-
ImageUtils.pullImages(dc.getDockerClient(),
24-
dc.getPullTimeout(),
25-
lifecycleImage.getReference());
23+
ImageUtils.pullImages(dc,lifecycleImage.getReference());
2624

2725
ImageInfo ii = ImageUtils.inspectImage(dc.getDockerClient(),
2826
lifecycleImage.getReference());

‎client/src/test/java/dev/snowdrop/buildpack/config/DockerConfigTest.java

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import static org.junit.jupiter.api.Assertions.assertNull;
77
import static org.junit.jupiter.api.Assertions.assertTrue;
88
import static org.mockito.Mockito.lenient;
9-
import static org.mockito.Mockito.when;
109

1110
import org.junit.jupiter.api.Test;
1211
import org.junit.jupiter.api.extension.ExtendWith;
@@ -20,20 +19,21 @@
2019
public class DockerConfigTest {
2120
@Test
2221
void checkTimeout() {
23-
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null);
22+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, null);
2423
assertEquals(60, dc1.getPullTimeout());
2524

26-
DockerConfig dc2 = new DockerConfig(245017, null, null, null, null, null);
25+
DockerConfig dc2 = new DockerConfig(245017, null, null, null, null, null, null, null, null);
2726
assertEquals(dc2.getPullTimeout(), 245017);
2827
}
2928

3029
@Test
3130
void checkDockerHost(@Mock DockerClient dockerClient, @Mock PingCmd pingCmd) {
32-
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null);
31+
lenient().when(dockerClient.pingCmd()).thenReturn(pingCmd);
32+
33+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, null);
3334
assertNotNull(dc1.getDockerHost());
3435

35-
when(dockerClient.pingCmd()).thenReturn(pingCmd);
36-
DockerConfig dc2 = new DockerConfig(null, "tcp://stilettos", null, null, null, dockerClient);
36+
DockerConfig dc2 = new DockerConfig(null, null, null, null, "tcp://stilettos", null, null, null, dockerClient);
3737
assertEquals("tcp://stilettos", dc2.getDockerHost());
3838
}
3939

@@ -42,48 +42,77 @@ void checkDockerSocket(@Mock DockerClient dockerClient, @Mock PingCmd pingCmd) {
4242

4343
lenient().when(dockerClient.pingCmd()).thenReturn(pingCmd);
4444

45-
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null);
45+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, null);
4646
assertNotNull(dc1.getDockerSocket());
4747

48-
DockerConfig dc2 = new DockerConfig(null, "unix:///stilettos", null, null, null, dockerClient);
48+
DockerConfig dc2 = new DockerConfig(null, null, null, null, "unix:///stilettos", null, null, null, dockerClient);
4949
assertEquals("/stilettos", dc2.getDockerSocket());
5050

51-
DockerConfig dc3 = new DockerConfig(null, "tcp://stilettos", null, null, null, dockerClient);
51+
DockerConfig dc3 = new DockerConfig(null, null, null, null, "tcp://stilettos", null, null, null, dockerClient);
5252
assertEquals("/var/run/docker.sock", dc3.getDockerSocket());
5353

54-
DockerConfig dc4 = new DockerConfig(null, null, "fish", null, null, null);
54+
DockerConfig dc4 = new DockerConfig(null, null, null, null, null, "fish", null, null, null);
5555
assertEquals("fish", dc4.getDockerSocket());
5656
}
5757

5858
@Test
5959
void checkDockerNetwork() {
60-
DockerConfig dc1 = new DockerConfig(null, null, null, "kitten", null, null);
60+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, "kitten", null, null);
6161
assertEquals("kitten", dc1.getDockerNetwork());
6262

63-
DockerConfig dc2 = new DockerConfig(null, null, null, null, null, null);
63+
DockerConfig dc2 = new DockerConfig(null, null, null, null, null, null, null, null, null);
6464
assertNull(dc2.getDockerNetwork());
6565
}
6666

6767
@Test
6868
void checkUseDaemon() {
69-
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null);
69+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, null);
7070
assertTrue(dc1.getUseDaemon());
7171

72-
DockerConfig dc2 = new DockerConfig(null, null, null, null, true, null);
72+
DockerConfig dc2 = new DockerConfig(null, null, null, null, null, null, null, true, null);
7373
assertTrue(dc2.getUseDaemon());
7474

75-
DockerConfig dc3 = new DockerConfig(null, null, null, null, false, null);
75+
DockerConfig dc3 = new DockerConfig(null, null, null, null, null, null, null, false, null);
7676
assertFalse(dc3.getUseDaemon());
7777
}
7878

7979
@Test
8080
void checkDockerClient(@Mock DockerClient dockerClient, @Mock PingCmd pingCmd){
8181
lenient().when(dockerClient.pingCmd()).thenReturn(pingCmd);
8282

83-
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null);
83+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, null);
8484
assertNotNull(dc1.getDockerClient());
8585

86-
DockerConfig dc2 = new DockerConfig(null, null, null, null, null, dockerClient);
86+
DockerConfig dc2 = new DockerConfig(null, null, null, null, null, null, null, null, dockerClient);
8787
assertEquals(dockerClient, dc2.getDockerClient());
8888
}
89+
90+
@Test
91+
void checkPullPolicy(@Mock DockerClient dockerClient, @Mock PingCmd pingCmd){
92+
lenient().when(dockerClient.pingCmd()).thenReturn(pingCmd);
93+
94+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, dockerClient);
95+
assertEquals(DockerConfig.PullPolicy.IF_NOT_PRESENT, dc1.getPullPolicy());
96+
97+
DockerConfig dc2 = new DockerConfig(null, null, null, DockerConfig.PullPolicy.IF_NOT_PRESENT, null, null, null, null, dockerClient);
98+
assertEquals(DockerConfig.PullPolicy.IF_NOT_PRESENT, dc2.getPullPolicy());
99+
100+
DockerConfig dc3 = new DockerConfig(null, null, null, DockerConfig.PullPolicy.ALWAYS, null, null, null, null, dockerClient);
101+
assertEquals(DockerConfig.PullPolicy.ALWAYS, dc3.getPullPolicy());
102+
}
103+
104+
105+
@Test
106+
void checkPullRetry(@Mock DockerClient dockerClient, @Mock PingCmd pingCmd){
107+
lenient().when(dockerClient.pingCmd()).thenReturn(pingCmd);
108+
109+
DockerConfig dc1 = new DockerConfig(null, null, null, null, null, null, null, null, dockerClient);
110+
assertEquals(3, dc1.getPullRetryCount());
111+
112+
DockerConfig dc2 = new DockerConfig(null, 5, null, null, null, null, null, null, dockerClient);
113+
assertEquals(5, dc2.getPullRetryCount());
114+
115+
DockerConfig dc3 = new DockerConfig(null, 0, null, null, null, null, null, null, dockerClient);
116+
assertEquals(0, dc3.getPullRetryCount());
117+
}
89118
}

0 commit comments

Comments
 (0)