Skip to content
This repository was archived by the owner on Nov 16, 2020. It is now read-only.

Commit a37e57b

Browse files
imikushinimikushin
authored
Use sourceDir and handler to build functions (#470)
* Use sourceDir and handler to build functions * Get rid of Exec when creating functions and building function images. * Build function images *before* calling faasDriver.Create(). * Write the source dir. * rm function.Code field. * Move TarDir and Untar to tar.go * Add TarGzBytes() to utils/tar.go * CLI: create function * Function.sourcePath for seed.yaml * TarDir to correctly normalize paths. * Migrate e2e tests. * `make check` shut up about mocks
1 parent b02309c commit a37e57b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+580
-767
lines changed

‎cmd/function-manager/main.go

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ import (
99
"os"
1010
"time"
1111

12+
docker "github.com/docker/docker/client"
1213
"github.com/go-openapi/loads"
1314
"github.com/go-openapi/loads/fmts"
1415
"github.com/go-openapi/swag"
1516
"github.com/jessevdk/go-flags"
1617
"github.com/justinas/alice"
1718
"github.com/opentracing/opentracing-go"
19+
"github.com/pkg/errors"
1820
log "github.com/sirupsen/logrus"
1921
"github.com/vmware/dispatch/pkg/client"
2022

@@ -36,12 +38,10 @@ import (
3638
"github.com/vmware/dispatch/pkg/utils"
3739
)
3840

39-
var drivers = map[string]func(string) functions.FaaSDriver{
40-
"openfaas": func(registryAuth string) functions.FaaSDriver {
41+
var drivers = map[string]func() functions.FaaSDriver{
42+
"openfaas": func() functions.FaaSDriver {
4143
faas, err := openfaas.New(&openfaas.Config{
4244
Gateway: config.Global.Function.OpenFaas.Gateway,
43-
ImageRegistry: config.Global.Registry.RegistryURI,
44-
RegistryAuth: registryAuth,
4545
K8sConfig: config.Global.Function.OpenFaas.K8sConfig,
4646
FuncNamespace: config.Global.Function.OpenFaas.FuncNamespace,
4747
FuncDefaultRequests: config.Global.Function.OpenFaas.FuncDefaultRequests,
@@ -53,10 +53,8 @@ var drivers = map[string]func(string) functions.FaaSDriver{
5353
}
5454
return faas
5555
},
56-
"riff": func(registryAuth string) functions.FaaSDriver {
56+
"riff": func() functions.FaaSDriver {
5757
faas, err := riff.New(&riff.Config{
58-
ImageRegistry: config.Global.Registry.RegistryURI,
59-
RegistryAuth: registryAuth,
6058
KafkaBrokers: config.Global.Function.Riff.KafkaBrokers,
6159
K8sConfig: config.Global.Function.Riff.K8sConfig,
6260
FuncNamespace: config.Global.Function.Riff.FuncNamespace,
@@ -68,7 +66,7 @@ var drivers = map[string]func(string) functions.FaaSDriver{
6866
}
6967
return faas
7068
},
71-
"openwhisk": func(registryAuth string) functions.FaaSDriver {
69+
"openwhisk": func() functions.FaaSDriver {
7270
faas, err := openwhisk.New(&openwhisk.Config{
7371
AuthToken: config.Global.Function.Openwhisk.AuthToken,
7472
Host: config.Global.Function.Openwhisk.Host,
@@ -79,10 +77,8 @@ var drivers = map[string]func(string) functions.FaaSDriver{
7977
}
8078
return faas
8179
},
82-
"kubeless": func(registryAuth string) functions.FaaSDriver {
80+
"kubeless": func() functions.FaaSDriver {
8381
faas, err := kubeless.New(&kubeless.Config{
84-
ImageRegistry: config.Global.Registry.RegistryURI,
85-
RegistryAuth: registryAuth,
8682
K8sConfig: config.Global.Function.Kubeless.K8sConfig,
8783
FuncNamespace: config.Global.Function.Kubeless.FuncNamespace,
8884
ImagePullSecret: config.Global.Function.Kubeless.ImagePullSecret,
@@ -92,11 +88,8 @@ var drivers = map[string]func(string) functions.FaaSDriver{
9288
}
9389
return faas
9490
},
95-
"noop": func(registryAuth string) functions.FaaSDriver {
96-
faas, err := noop.New(&noop.Config{
97-
ImageRegistry: config.Global.Registry.RegistryURI,
98-
RegistryAuth: registryAuth,
99-
})
91+
"noop": func() functions.FaaSDriver {
92+
faas, err := noop.New(&noop.Config{})
10093
if err != nil {
10194
log.Fatalf("Error starting noop driver: %+v", err)
10295
}
@@ -180,7 +173,7 @@ func main() {
180173
log.Fatalln(err)
181174
}
182175

183-
faas := drivers[config.Global.Function.Faas](registryAuth)
176+
faas := drivers[config.Global.Function.Faas]()
184177
defer utils.Close(faas)
185178

186179
c := &functionmanager.ControllerConfig{
@@ -203,7 +196,14 @@ func main() {
203196
if config.Global.Function.FileImageManager != "" {
204197
imageGetter = functionmanager.FileImageManagerClient()
205198
}
206-
controller := functionmanager.NewController(c, es, faas, r, imageGetter)
199+
200+
dc, err := docker.NewEnvClient()
201+
if err != nil {
202+
log.Fatalln(errors.Wrap(err, "could not get docker client"))
203+
}
204+
imageBuilder := functions.NewDockerImageBuilder(config.Global.Registry.RegistryURI, registryAuth, dc)
205+
206+
controller := functionmanager.NewController(c, es, faas, r, imageGetter, imageBuilder)
207207
defer controller.Shutdown()
208208
controller.Start()
209209

‎docs/_specs/language-support/language-support.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ In order to create the *image*, **Dispatch image manager** does the following:
149149

150150
A **user** creates a *function* from a source file using the specified *image*:
151151

152-
$ dispatch create function js-deps hello1 ./hello.js
152+
$ dispatch create function --image=js-deps hello1 . --handler=./hello.js
153153

154154
Here, the *function* named `hello1` is created from the source file `./hello.js` using the image `js-deps` containing library packages that can be used by the *function*.
155155

‎e2e/tests/apis.bats

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ load variables
2020
}
2121

2222
@test "Create Functions for test" {
23-
run dispatch create function nodejs6 func-nodejs6 ${DISPATCH_ROOT}/examples/nodejs6/hello.js
23+
run dispatch create function --image=nodejs6 func-nodejs6 ${DISPATCH_ROOT}/examples/nodejs6 --handler=./hello.js
2424
echo_to_log
2525
assert_success
2626

2727
run_with_retry "dispatch get function func-nodejs6 --json | jq -r .status" "READY" 10 5
2828

29-
run dispatch create function nodejs6 node-echo-back ${DISPATCH_ROOT}/examples/nodejs6/debug.js
29+
run dispatch create function --image=nodejs6 node-echo-back ${DISPATCH_ROOT}/examples/nodejs6 --handler=./debug.js
3030
echo_to_log
3131
assert_success
3232

‎e2e/tests/application.bats

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ load variables
7474

7575
@test "Function creation" {
7676

77-
run dispatch create function foo-app-image foo-app-func -a foo-app ${DISPATCH_ROOT}/examples/nodejs6/i-have-a-secret.js --secret open-sesame
77+
run dispatch create function --image=foo-app-image foo-app-func -a foo-app ${DISPATCH_ROOT}/examples/nodejs6 --handler=./i-have-a-secret.js --secret open-sesame
7878
echo_to_log
7979
assert_success
8080

‎e2e/tests/events.bats

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ load variables
1313
func_name=node-echo-back-${RANDOM}
1414
sub_name=testsub-${RANDOM}
1515
event_name=test.event.${RANDOM}
16-
run dispatch create function nodejs6 ${func_name} ${DISPATCH_ROOT}/examples/nodejs6/debug.js
16+
run dispatch create function --image=nodejs6 ${func_name} ${DISPATCH_ROOT}/examples/nodejs6 --handler=./debug.js
1717
echo_to_log
1818
assert_success
1919

@@ -82,7 +82,7 @@ load variables
8282
sub_name=testsub-${RANDOM}
8383
driver_name=testdriver-${RANDOM}
8484

85-
run dispatch create function nodejs6 ${func_name} ${DISPATCH_ROOT}/examples/nodejs6/debug.js
85+
run dispatch create function --image=nodejs6 ${func_name} ${DISPATCH_ROOT}/examples/nodejs6 --handler=./debug.js
8686
echo_to_log
8787
assert_success
8888

‎e2e/tests/function_update.yaml

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,10 @@
1-
code: |
2-
#!/usr/bin/env python
3-
#######################################################################
4-
## Copyright (c) 2017 VMware, Inc. All Rights Reserved.
5-
## SPDX-License-Identifier: Apache-2.0
6-
#######################################################################
7-
"""
8-
Example function "Hello World"
9-
10-
** REQUIREMENTS **
11-
12-
* image
13-
dispatch create base-image python3 dispatchframework/python3-base:0.0.3 --language python3
14-
dispatch create image python3 python3
15-
16-
Create a function:
17-
dispatch create function python3 hello-python examples/python3/hello.py
18-
19-
Execute it:
20-
dispatch exec hello-python --wait --input='{"name": "Jon", "place": "Winterfell"}'
21-
22-
"""
23-
24-
def handle(ctx, payload):
25-
name = "Noone"
26-
place = "Nowhere"
27-
if payload:
28-
name = payload.get("name", name)
29-
place = payload.get("place", place)
30-
return {"myField": "Goodbye, %s from %s" % (name, place)}
311
image: python3
322
kind: Function
333
name: python-hello-no-schema
4+
sourcePath: update-source
5+
handler: hello.handle
346
schema: {}
357
secrets:
368
tags:
379
- key: update
38-
value: tags
10+
value: tags

‎e2e/tests/functions.bats

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,19 @@ load variables
1111

1212
@test "Create node function no schema" {
1313

14-
run dispatch create function nodejs6 node-hello-no-schema ${DISPATCH_ROOT}/examples/nodejs6/hello.js
14+
run dispatch create function --image=nodejs6 node-hello-no-schema ${DISPATCH_ROOT}/examples/nodejs6 --handler=./hello.js
1515
echo_to_log
1616
assert_success
1717

1818
run_with_retry "dispatch get function node-hello-no-schema --json | jq -r .status" "READY" 8 5
1919
}
2020

2121
@test "Create a function with duplicate name" {
22-
run dispatch create function nodejs6 node-hello-dup ${DISPATCH_ROOT}/examples/nodejs6/hello.js
22+
run dispatch create function --image=nodejs6 node-hello-dup ${DISPATCH_ROOT}/examples/nodejs6 --handler=./hello.js
2323
echo_to_log
2424
assert_success
2525

26-
run dispatch create function nodejs6 node-hello-dup ${DISPATCH_ROOT}/examples/nodejs6/hello.js
26+
run dispatch create function --image=nodejs6 node-hello-dup ${DISPATCH_ROOT}/examples/nodejs6 --handler=./hello.js
2727
assert_failure
2828
}
2929

@@ -40,7 +40,7 @@ load variables
4040
}
4141

4242
@test "Create python function no schema" {
43-
run dispatch create function python3 python-hello-no-schema ${DISPATCH_ROOT}/examples/python3/hello.py
43+
run dispatch create function --image=python3 python-hello-no-schema ${DISPATCH_ROOT}/examples/python3 --handler=hello.handle
4444
echo_to_log
4545
assert_success
4646

@@ -52,7 +52,7 @@ load variables
5252
}
5353

5454
@test "Create powershell function no schema" {
55-
run dispatch create function powershell powershell-hello-no-schema ${DISPATCH_ROOT}/examples/powershell/hello.ps1
55+
run dispatch create function --image=powershell powershell-hello-no-schema ${DISPATCH_ROOT}/examples/powershell --handler=hello.ps1::handle
5656
echo_to_log
5757
assert_success
5858

@@ -68,7 +68,7 @@ load variables
6868
assert_success
6969
run_with_retry "dispatch get image powershell-with-slack --json | jq -r .status" "READY" 10 5
7070

71-
run dispatch create function powershell-with-slack powershell-slack ${DISPATCH_ROOT}/examples/powershell/test-slack.ps1
71+
run dispatch create function --image=powershell-with-slack powershell-slack ${DISPATCH_ROOT}/examples/powershell --handler=test-slack.ps1::handle
7272
echo_to_log
7373
assert_success
7474

@@ -80,7 +80,7 @@ load variables
8080
}
8181

8282
@test "Create java function no schema" {
83-
run dispatch create function java8 java-hello-no-schema ${DISPATCH_ROOT}/examples/java8/Hello.java
83+
run dispatch create function --image=java java-hello-no-schema ${DISPATCH_ROOT}/examples/java/hello-with-deps --handler=io.dispatchframework.examples.Hello
8484
echo_to_log
8585
assert_success
8686

@@ -92,11 +92,11 @@ load variables
9292
}
9393

9494
@test "Create java function with runtime deps" {
95-
run dispatch create image java8-with-deps java8-base --runtime-deps ${DISPATCH_ROOT}/examples/java8/pom.xml
95+
run dispatch create image java-with-deps java-base --runtime-deps ${DISPATCH_ROOT}/examples/java/hello-with-deps/pom.xml
9696
assert_success
97-
run_with_retry "dispatch get image java8-with-deps --json | jq -r .status" "READY" 20 5
97+
run_with_retry "dispatch get image java-with-deps --json | jq -r .status" "READY" 20 5
9898

99-
run dispatch create function java8-with-deps java-hello-with-deps ${DISPATCH_ROOT}/examples/java8/HelloWithDeps.java
99+
run dispatch create function --image=java-with-deps java-hello-with-deps ${DISPATCH_ROOT}/examples/java/hello-with-deps --handler=io.dispatchframework.examples.HelloWithDeps
100100
echo_to_log
101101
assert_success
102102

@@ -108,7 +108,7 @@ load variables
108108
}
109109

110110
@test "Create java function with classes" {
111-
run dispatch create function java8 java-hello-with-classes ${DISPATCH_ROOT}/examples/java8/HelloWithClasses.java
111+
run dispatch create function --image=java java-hello-with-classes ${DISPATCH_ROOT}/examples/java/hello-with-deps --handler=io.dispatchframework.examples.HelloWithClasses
112112
echo_to_log
113113
assert_success
114114

@@ -120,12 +120,12 @@ load variables
120120
}
121121

122122
@test "Create java function with spring support" {
123-
run dispatch create image java8-spring java8-base --runtime-deps ${DISPATCH_ROOT}/examples/java8/spring-pom.xml
123+
run dispatch create image java-spring java-base --runtime-deps ${DISPATCH_ROOT}/examples/java/spring-pom.xml
124124
assert_success
125125

126-
run_with_retry "dispatch get image java8-spring --json | jq -r .status" "READY" 10 5
126+
run_with_retry "dispatch get image java-spring --json | jq -r .status" "READY" 10 5
127127

128-
run dispatch create function java8-spring spring-fn ${DISPATCH_ROOT}/examples/java8/HelloSpring.java
128+
run dispatch create function --image=java-spring spring-fn ${DISPATCH_ROOT}/examples/java/hello-with-deps --handler=io.dispatchframework.examples.HelloSpring
129129
echo_to_log
130130
assert_success
131131

@@ -137,7 +137,7 @@ load variables
137137
}
138138

139139
@test "Create node function with schema" {
140-
run dispatch create function nodejs6 node-hello-with-schema ${DISPATCH_ROOT}/examples/nodejs6/hello.js --schema-in ${DISPATCH_ROOT}/examples/nodejs6/hello.schema.in.json --schema-out ${DISPATCH_ROOT}/examples/nodejs6/hello.schema.out.json
140+
run dispatch create function --image=nodejs6 node-hello-with-schema ${DISPATCH_ROOT}/examples/nodejs6 --handler=./hello.js --schema-in ${DISPATCH_ROOT}/examples/nodejs6/hello.schema.in.json --schema-out ${DISPATCH_ROOT}/examples/nodejs6/hello.schema.out.json
141141
echo_to_log
142142
assert_success
143143

@@ -157,7 +157,7 @@ load variables
157157
}
158158

159159
@test "Create python function with runtime deps" {
160-
run dispatch create function python3 http ${DISPATCH_ROOT}/examples/python3/http.py
160+
run dispatch create function --image=python3 http ${DISPATCH_ROOT}/examples/python3 --handler=http_func.handle
161161
echo_to_log
162162
assert_success
163163

@@ -169,16 +169,16 @@ load variables
169169
}
170170

171171
@test "Create python function with logging" {
172-
src_file=$(mktemp).py
173-
cat << EOF > ${src_file}
172+
src_dir=$(mktemp -d)
173+
cat << EOF > ${src_dir}/logging_test.py
174174
import sys
175175
176176
def handle(ctx, payload):
177177
print("this goes to stdout")
178178
print("this goes to stderr", file=sys.stderr)
179179
EOF
180180

181-
run dispatch create function python3 logger ${src_file}
181+
run dispatch create function --image=python3 logger ${src_dir} --handler=logging_test.handle
182182
echo_to_log
183183
assert_success
184184

‎e2e/tests/images.bats

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ load variables
3333
run_with_retry "dispatch get base-image base-powershell --json | jq -r .status" "INITIALIZED" 1 0
3434
run_with_retry "dispatch get base-image base-powershell --json | jq -r .status" "READY" 10 5
3535

36-
# Create base image "base-java8"
37-
run dispatch create base-image base-java8 $DOCKER_REGISTRY/$BASE_IMAGE_JAVA8 --language java8
36+
# Create base image "base-java"
37+
run dispatch create base-image base-java $DOCKER_REGISTRY/$BASE_IMAGE_JAVA --language java
3838
assert_success
3939

40-
run_with_retry "dispatch get base-image base-java8 --json | jq -r .status" "INITIALIZED" 1 0
41-
run_with_retry "dispatch get base-image base-java8 --json | jq -r .status" "READY" 10 5
40+
run_with_retry "dispatch get base-image base-java --json | jq -r .status" "INITIALIZED" 1 0
41+
run_with_retry "dispatch get base-image base-java --json | jq -r .status" "READY" 10 5
4242

4343
# Create fifth image with non-existing image. Check that get operation returns five images. Wait for "ERROR" status for missing image.
4444
run dispatch create base-image missing-image missing/image:latest --language nodejs6
@@ -55,12 +55,12 @@ load variables
5555
assert_success
5656
run dispatch create image powershell base-powershell
5757
assert_success
58-
run dispatch create image java8 base-java8
58+
run dispatch create image java base-java
5959
assert_success
6060
run_with_retry "dispatch get image nodejs6 --json | jq -r .status" "READY" 8 5
6161
run_with_retry "dispatch get image python3 --json | jq -r .status" "READY" 8 5
6262
run_with_retry "dispatch get image powershell --json | jq -r .status" "READY" 8 5
63-
run_with_retry "dispatch get image java8 --json | jq -r .status" "READY" 8 5
63+
run_with_retry "dispatch get image java --json | jq -r .status" "READY" 8 5
6464
}
6565

6666
@test "Delete images" {

0 commit comments

Comments
 (0)