Service manager initial commit (#346)

* WIP: OSBAPI support for service integration

* Update vendored files

* vendor riff separately... ugh

* fix merge issues

* fix swagger generated files

* regenerate swagger files

* fix postgres driver and bump e2e retries

* Allow objects passed as parameters

* previously just allowed map[string]string which doesn't work
  for all brokers
* add some more tests
* slight cleanup

Author: berndtj

Gopkg.lock

@@ -57,38 +57,26 @@
   name = "github.com/tylerb/graceful"
   version = "1.2.15"
 
-[[constraint]]
-  name = "github.com/ugorji/go"
-  revision = "5dfac818335991034af249bfc598e41c73ea1826"
-
 [[constraint]]
   name = "k8s.io/api"
-  branch = "release-1.8"
+  version = "kubernetes-1.9.1"
 
 [[constraint]]
   name = "k8s.io/apimachinery"
-  branch = "release-1.8"
+  version = "kubernetes-1.9.1"
 
 [[constraint]]
   name = "k8s.io/client-go"
-  branch = "release-5.0"
+  version = "kubernetes-1.9.1"
 
 [[constraint]]
   name = "k8s.io/code-generator"
-  branch = "release-1.8"
-
-[[override]]
-  name = "k8s.io/gengo"
-  revision = "9e661e9308f078838e266cca1c673922088c0ea4"
+  version = "kubernetes-1.9.1"
 
 [[constraint]]
   name = "github.com/sirupsen/logrus"
   version = "v1.0.3"
 
-[[override]]
-  name = "github.com/ugorji/go"
-  revision = "8c0409fcbb70099c748d71f714529204975f6c3f"
-
 [[constraint]]
   name = "github.com/streadway/amqp"
   branch = "master"

Gopkg.toml

@@ -42,7 +42,7 @@ all: generate linux darwin
 .PHONY: goversion
 goversion:
 	@echo Checking go version...
-	@(goversion=$$($(GO) version | cut -d' ' -f3); echo "$(GOVERSIONS)" | grep -q $$goversion || ( echo "Please install one of $(GOVERSIONS) (found: $$goversion)" && exit 2 ))
+	@(goversion=$$($(GO) version | cut -d' ' -f3 | cut -d'.' -f1,2); echo "$(GOVERSIONS)" | grep -q $$goversion || ( echo "Please install one of $(GOVERSIONS) (found: $$goversion)" && exit 2 ))
 
 .PHONY: check
 check: goversion checkfmt checklint swagger-validate ## check if the source files comply to the formatting rules
@@ -95,7 +95,8 @@ run-dev: ## run the dev server
 
 CLI = dispatch
 SERVICES = api-manager application-manager event-driver event-manager \
-           function-manager identity-manager image-manager secret-store event-sidecar
+           function-manager identity-manager image-manager secret-store event-sidecar \
+           service-manager
 
 DARWIN_BINS = $(CLI)-darwin $(foreach bin,$(SERVICES),$(bin)-darwin)
 LINUX_BINS = $(CLI)-linux $(foreach bin,$(SERVICES),$(bin)-linux)
@@ -138,6 +139,7 @@ generate: ## run go generate
 	scripts/generate.sh identity-manager IdentityManager identity-manager.yaml
 	scripts/generate.sh image-manager ImageManager image-manager.yaml
 	scripts/generate.sh secret-store SecretStore secret-store.yaml
+	scripts/generate.sh service-manager ServiceManager service-manager.yaml
 	scripts/header-check.sh fix
 
 .PHONY: gen-clean

Makefile

@@ -0,0 +1,155 @@
+///////////////////////////////////////////////////////////////////////
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+///////////////////////////////////////////////////////////////////////
+
+package main
+
+import (
+	"os"
+	"time"
+
+	"github.com/go-openapi/loads"
+	"github.com/go-openapi/loads/fmts"
+	"github.com/go-openapi/swag"
+	"github.com/jessevdk/go-flags"
+	"github.com/justinas/alice"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vmware/dispatch/pkg/config"
+	"github.com/vmware/dispatch/pkg/entity-store"
+	"github.com/vmware/dispatch/pkg/middleware"
+	"github.com/vmware/dispatch/pkg/service-manager"
+	"github.com/vmware/dispatch/pkg/service-manager/clients"
+	servicemanagerflags "github.com/vmware/dispatch/pkg/service-manager/flags"
+	"github.com/vmware/dispatch/pkg/service-manager/gen/restapi"
+	"github.com/vmware/dispatch/pkg/service-manager/gen/restapi/operations"
+	"github.com/vmware/dispatch/pkg/trace"
+)
+
+func init() {
+	loads.AddLoader(fmts.YAMLMatcher, fmts.YAMLDoc)
+}
+
+var debugFlags = struct {
+	DebugEnabled   bool `long:"debug" description:"Enable debugging messages"`
+	TracingEnabled bool `long:"trace" description:"Enable tracing messages (enables debugging)"`
+}{}
+
+func configureFlags() []swag.CommandLineOptionsGroup {
+	return []swag.CommandLineOptionsGroup{
+		{
+			ShortDescription: "Service manager Flags",
+			LongDescription:  "",
+			Options:          &servicemanagerflags.ServiceManagerFlags,
+		},
+		{
+			ShortDescription: "Debug options",
+			LongDescription:  "",
+			Options:          &debugFlags,
+		},
+	}
+}
+
+func main() {
+	swaggerSpec, err := loads.Analyzed(restapi.SwaggerJSON, "2.0")
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	api := operations.NewServiceManagerAPI(swaggerSpec)
+	server := restapi.NewServer(api)
+
+	parser := flags.NewParser(server, flags.Default)
+	parser.ShortDescription = "Service Manager"
+	parser.LongDescription = "This is the API server for the Dispatch Service Manager service.\n"
+
+	optsGroups := configureFlags()
+	for _, optsGroup := range optsGroups {
+		_, err := parser.AddGroup(optsGroup.ShortDescription, optsGroup.LongDescription, optsGroup.Options)
+		if err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if _, err := parser.Parse(); err != nil {
+		code := 1
+		if fe, ok := err.(*flags.Error); ok {
+			if fe.Type == flags.ErrHelp {
+				code = 0
+			}
+		}
+		os.Exit(code)
+	}
+
+	if debugFlags.DebugEnabled {
+		log.SetLevel(log.DebugLevel)
+	}
+	if debugFlags.TracingEnabled {
+		log.SetLevel(log.DebugLevel)
+		trace.Enable()
+	}
+
+	config.Global = config.LoadConfiguration(servicemanagerflags.ServiceManagerFlags.Config)
+
+	store, err := entitystore.NewFromBackend(
+		entitystore.BackendConfig{
+			Backend:  servicemanagerflags.ServiceManagerFlags.DbBackend,
+			Address:  servicemanagerflags.ServiceManagerFlags.DbFile,
+			Bucket:   servicemanagerflags.ServiceManagerFlags.DbDatabase,
+			Username: servicemanagerflags.ServiceManagerFlags.DbUser,
+			Password: servicemanagerflags.ServiceManagerFlags.DbPassword,
+		})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	k8sClient, err := clients.NewK8sBrokerClient(
+		clients.K8sBrokerConfigOpts{
+			K8sConfig:        servicemanagerflags.ServiceManagerFlags.K8sConfig,
+			CatalogNamespace: servicemanagerflags.ServiceManagerFlags.K8sCatalogNamespace,
+			SecretStoreURL:   servicemanagerflags.ServiceManagerFlags.SecretStore,
+			OrgID:            servicemanagerflags.ServiceManagerFlags.OrgID,
+		},
+	)
+	if err != nil {
+		log.Fatalf("Error creating k8sClient: %v", err)
+	}
+	// service controller
+	serviceController := servicemanager.NewController(
+		&servicemanager.ControllerConfig{
+			OrganizationID: servicemanagerflags.ServiceManagerFlags.OrgID,
+			ResyncPeriod:   time.Second * time.Duration(servicemanagerflags.ServiceManagerFlags.ResyncPeriod),
+		},
+		store,
+		k8sClient,
+	)
+
+	defer serviceController.Shutdown()
+	serviceController.Start()
+
+	// handler
+	handlers := &servicemanager.Handlers{
+		Store:   store,
+		Watcher: serviceController.Watcher(),
+	}
+
+	handlers.ConfigureHandlers(api)
+
+	healthChecker := func() error {
+		// TODO: implement service-specific healthchecking
+		return nil
+	}
+
+	handler := alice.New(
+		middleware.NewHealthCheckMW("", healthChecker),
+	).Then(api.Serve(nil))
+
+	server.SetHandler(handler)
+
+	defer server.Shutdown()
+	if err := server.Serve(); err != nil {
+		log.Fatalln(err)
+	}
+
+}

cmd/service-manager/main.go

@@ -78,7 +78,7 @@ load variables
     echo_to_log
     assert_success
 
-    run_with_retry "dispatch get function -a foo-app --json | jq -r .[].status" "READY" 8 5
+    run_with_retry "dispatch get function -a foo-app --json | jq -r .[].status" "READY" 20 5
 }
 
 @test "Function execution" {

e2e/tests/application.bats

@@ -0,0 +1,9 @@
+FROM vmware/photon:2.0
+
+ADD bin/service-manager-linux /service-manager
+RUN chmod +x /service-manager
+
+VOLUME ["/data/service-manager"]
+
+ENTRYPOINT ["/service-manager"]
+CMD ["--db-file=/data/service-manager/db.bolt", "--host=0.0.0.0", "--port=8000"]