WIP: images are now built not just pass-through (#167)

This is the first step towards being able to build images with
dependencies.  We now build a docker image based on the base-image.
There are other cool things in this commit, such as being able
to use a local repository (yay!) `helm install stable/docker-registry`.

* Added docker in docker side-cars to image manager and function
  manager so that they can reach local repos within the k8s cluster
  - Set DOCKER_HOST to the localhost socket as opposed to the
    host's unix socket
* Build docker images based on the base-image
* Unify the repostory config (do we need separate repo configs for
  FaaSs and image manager?)
* Function manager is more async now as images are not necessarily
  "ready" when function created.
  - Should make "sync" smarter...
* Tried to generalize the image building, but I haven't changed the
  function manager code to use it... yet.
* Image manager now uses global config.

Testing Done:
- Installed and ran chart
- Needs more tests and e2e

Author: berndtj

README.md

@@ -111,7 +111,7 @@ Make a note of the `<port>` as this will be used in the next step.
 Dispatch runs on a non-standard https port on minikube since it uses
 NodePort for the ingress controller service. Hence, update the
 Authorization callback URL of your OAuth2 Client App (created by following
-[How to Create OAuth Client App](docs/create-oauth-client-app.md)) from
+[How to Create OAuth Client App](docs/_guides/create-oauth-client-app.md)) from
 `https://<DISPATCH_HOST>/oauth2` to `https://<DISPATCH_HOST>:<port>/oauth2`
 where `<port>` can be found in your dispatch config file available at
 $HOME/.dispatch/config.json.

charts/dispatch/charts/application-manager/templates/ingress.yaml

@@ -31,7 +31,9 @@ spec:
   {{- if $tls.secretName }}
   tls:
     - secretName: {{ $tls.secretName }}
+      {{- if $ingress_host }}
       hosts:
         - {{ default .Values.global.host .Values.ingress.host }}
+      {{- end -}}
   {{- end -}}
 {{- end -}}

charts/dispatch/charts/function-manager/templates/config-map.yaml

@@ -8,12 +8,13 @@ data:
   config.json: |-
     {
       "openwhisk": {
-        "auth_token": "{{ .Values.faas.openwhisk.authToken }}",
         "host": "{{ .Values.faas.openwhisk.host }}"
       },
       "openfaas": {
-        "gateway": "{{ .Values.faas.openfaas.gateway }}",
-        "image_registry": "{{ .Values.faas.openfaas.imageRegistry }}",
-        "registry_auth": "{{ .Values.faas.openfaas.registryAuth }}"
+        "gateway": "{{ .Values.faas.openfaas.gateway }}"
+      },
+      "registry": {
+        "uri": "{{ default .Values.global.registry.uri .Values.registry.uri }}",
+        "auth": "{{ default .Values.global.registry.auth .Values.registry.auth }}"
       }
     }

charts/dispatch/charts/function-manager/templates/deployment.yaml

@@ -74,21 +74,33 @@ spec:
             - mountPath: /data/config
               name: {{ template "fullname" . }}-config
               readOnly: true
-            - mountPath: "/var/run/docker.sock"
-              name: {{ template "fullname" . }}-docker
             - mountPath: "/data/tls"
               name: tls
               readOnly: true
           env:
             - name: DOCKER_API_VERSION
               value: "1.24"
+            - name: DOCKER_HOST
+              value: "tcp://localhost:2375"
             - name: ORGANIZATION
               valueFrom:
                 configMapKeyRef:
                   name: {{ template "fullname" . }}
                   key: organization
           resources:
 {{ toYaml .Values.resources | indent 12 }}
+        - name: {{ .Chart.Name }}-docker
+          image: docker:dind
+          imagePullPolicy: {{ default .Values.global.pullPolicy .Values.image.pullPolicy }}
+          {{- if default .Values.global.registry.insecure .Values.registry.insecure }}
+          args:
+          - --insecure-registry={{ default .Values.global.registry.uri .Values.registry.uri }}
+          {{- end }}
+          securityContext:
+            privileged: true
+          volumeMounts:
+          - name: docker-graph-storage
+            mountPath: /var/lib/docker
       volumes:
         - name: {{ template "fullname" . }}
 {{- if default .Values.global.data.persist .Values.data.persist }}
@@ -103,12 +115,11 @@ spec:
             items:
             - key: config.json
               path: {{ template "name" . }}.json
-        - name: {{ template "fullname" . }}-docker
-          hostPath:
-            path: /var/run/docker.sock
         - name: tls
           secret:
             secretName: dispatch-tls
+        - name: docker-graph-storage
+          emptyDir: {}
 {{- if .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}

charts/dispatch/charts/function-manager/values.yaml

@@ -42,8 +42,9 @@ faas:
     host: "52.91.175.16"
   openfaas:
     gateway: "http://gateway.openfaas:8080/"
-    imageRegistry: "vmware"
-    registryAuth: SECRET
+registry: {}
+  # insecure: true
+  # uri: docker-docker-registry.docker.svc.cluster.local:5000
 data:
   # persist: false
   hostPath: /var/function-manager

charts/dispatch/charts/image-manager/templates/config-map.yaml

@@ -5,5 +5,10 @@ metadata:
   namespace: {{ .Release.Namespace }}
 data:
   organization: {{ .Values.global.organization }}
-  serverless.json: |-
-    {}
+  config.json: |-
+    {
+      "registry": {
+        "uri": "{{ default .Values.global.registry.uri .Values.registry.uri }}",
+        "auth": "{{ default .Values.global.registry.auth .Values.registry.auth }}"
+      }
+    }

charts/dispatch/charts/image-manager/templates/deployment.yaml

@@ -26,6 +26,7 @@ spec:
           image: "{{ default .Values.global.image.host .Values.image.host }}/{{ .Values.image.repository }}:{{ default .Values.global.image.tag .Values.image.tag }}"
           imagePullPolicy: {{ default .Values.global.pullPolicy .Values.image.pullPolicy }}
           args:
+            - "--config=/data/config/{{ template "name" . }}.json"
             - "--organization=$(ORGANIZATION)"
             - "--host=0.0.0.0"
             - "--port={{ .Values.service.internalPort }}"
@@ -66,21 +67,36 @@ spec:
           volumeMounts:
             - mountPath: "/data/{{ template "name" . }}"
               name: {{ template "fullname" . }}
-            - mountPath: "/var/run/docker.sock"
-              name: {{ template "fullname" . }}-docker
+            - mountPath: /data/config
+              name: {{ template "fullname" . }}-config
+              readOnly: true
             - mountPath: "/data/tls"
               name: tls
               readOnly: true
           env:
             - name: DOCKER_API_VERSION
               value: "1.24"
+            - name: DOCKER_HOST
+              value: "tcp://localhost:2375"
             - name: ORGANIZATION
               valueFrom:
                 configMapKeyRef:
                   name: {{ template "fullname" . }}
                   key: organization
           resources:
 {{ toYaml .Values.resources | indent 12 }}
+        - name: {{ .Chart.Name }}-docker
+          image: docker:dind
+          imagePullPolicy: {{ default .Values.global.pullPolicy .Values.image.pullPolicy }}
+          {{- if default .Values.global.registry.insecure .Values.registry.insecure }}
+          args:
+          - --insecure-registry={{ default .Values.global.registry.uri .Values.registry.uri }}
+          {{- end }}
+          securityContext:
+            privileged: true
+          volumeMounts:
+          - name: docker-graph-storage
+            mountPath: /var/lib/docker
       volumes:
         - name: {{ template "fullname" . }}
 {{- if default .Values.global.data.persist .Values.data.persist }}
@@ -89,12 +105,17 @@ spec:
 {{- else }}
           emptyDir: {}
 {{- end }}
-        - name: {{ template "fullname" . }}-docker
-          hostPath:
-            path: /var/run/docker.sock
+        - name: {{ template "fullname" . }}-config
+          configMap:
+            name: {{ template "fullname" . }}
+            items:
+            - key: config.json
+              path: {{ template "name" . }}.json
         - name: tls
           secret:
             secretName: dispatch-tls
+        - name: docker-graph-storage
+          emptyDir: {}
 {{- if .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}

charts/dispatch/charts/image-manager/values.yaml

@@ -35,6 +35,9 @@ resources: {}
   #requests:
   #  cpu: 100m
   #  memory: 128Mi
+registry: {}
+  # insecure: true
+  # uri: docker-docker-registry.docker.svc.cluster.local:5000
 data:
   # persist: false
   hostPath: /var/image-manager

charts/dispatch/values.yaml

@@ -5,7 +5,7 @@ global:
   pullPolicy: IfNotPresent
   organization: dispatch
   image:
-    tag: v0.1.2
+    tag: v0.1.3
     host: vmware
   debug: true
   trace: false
@@ -25,5 +25,8 @@ global:
     database: dispatch
     namespace: dispatch
     release: postgres
+  registry:
+    insecure: true
+    uri: docker-docker-registry.docker.svc.cluster.local:5000
 rabbitmq:
   rabbitmqPassword: serverless

cmd/function-manager/main.go

@@ -32,23 +32,23 @@ import (
 	"github.com/vmware/dispatch/pkg/trace"
 )
 
-var drivers = map[string]func() functions.FaaSDriver{
-	"openfaas": func() functions.FaaSDriver {
+var drivers = map[string]func(string) functions.FaaSDriver{
+	"openfaas": func(registryAuth string) functions.FaaSDriver {
 		faas, err := openfaas.New(&openfaas.Config{
 			Gateway:       config.Global.OpenFaas.Gateway,
-			ImageRegistry: config.Global.OpenFaas.ImageRegistry,
-			RegistryAuth:  config.Global.OpenFaas.RegistryAuth,
+			ImageRegistry: config.Global.Registry.RegistryURI,
+			RegistryAuth:  registryAuth,
 		})
 		if err != nil {
 			log.Fatalf("Error starting OpenFaaS driver: %+v", err)
 		}
 		return faas
 	},
-	"riff": func() functions.FaaSDriver {
+	"riff": func(registryAuth string) functions.FaaSDriver {
 		faas, err := riff.New(&riff.Config{
-			ImageRegistry: config.Global.Riff.ImageRegistry,
+			ImageRegistry: config.Global.Registry.RegistryURI,
+			RegistryAuth:  registryAuth,
 			Gateway:       config.Global.Riff.Gateway,
-			RegistryAuth:  config.Global.Riff.RegistryAuth,
 			K8sConfig:     config.Global.Riff.K8sConfig,
 			RiffNamespace: config.Global.Riff.RiffNamespace,
 		})
@@ -57,7 +57,7 @@ var drivers = map[string]func() functions.FaaSDriver{
 		}
 		return faas
 	},
-	"openwhisk": func() functions.FaaSDriver {
+	"openwhisk": func(registryAuth string) functions.FaaSDriver {
 		faas, err := openwhisk.New(&openwhisk.Config{
 			AuthToken: config.Global.Openwhisk.AuthToken,
 			Host:      config.Global.Openwhisk.Host,
@@ -132,6 +132,11 @@ func main() {
 
 	config.Global = config.LoadConfiguration(functionmanager.FunctionManagerFlags.Config)
 
+	registryAuth := config.Global.Registry.RegistryAuth
+	if config.Global.Registry.RegistryAuth == "" {
+		registryAuth = config.EmptyRegistryAuth
+	}
+
 	es, err := entitystore.NewFromBackend(
 		entitystore.BackendConfig{
 			Backend:  functionmanager.FunctionManagerFlags.DbBackend,
@@ -144,7 +149,7 @@ func main() {
 		log.Fatalln(err)
 	}
 
-	faas := drivers[functionmanager.FunctionManagerFlags.Faas]()
+	faas := drivers[functionmanager.FunctionManagerFlags.Faas](registryAuth)
 
 	c := &functionmanager.ControllerConfig{
 		ResyncPeriod:   time.Duration(functionmanager.FunctionManagerFlags.ResyncPeriod) * time.Second,