Add databricks labs command group (#914)

## Command group
<img width="688" alt="image"
src="https://github.com/databricks/cli/assets/259697/51a3d309-2244-40ff-b6c3-4f151da6a6ec">

## Installed versions
<img width="388" alt="image"
src="https://github.com/databricks/cli/assets/259697/0873e8ac-20cc-4bab-bb32-e064eddc05f2">

## Project commands
<img width="479" alt="image"
src="https://github.com/databricks/cli/assets/259697/618949e2-99f1-466b-9288-97e88c715518">

## Installer hook

![image](https://github.com/databricks/cli/assets/259697/3ce0d355-039a-445f-bff7-6dfc1a2e3288)

## Update notifications

![image](https://github.com/databricks/cli/assets/259697/10724627-3606-49e1-9722-00ae37afed12)

# Downstream work

- databrickslabs/ucx#517
- databrickslabs/dlt-meta#19
- databrickslabs/discoverx#84

Author: nfx

cmd/cmd.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/cmd/bundle"
 	"github.com/databricks/cli/cmd/configure"
 	"github.com/databricks/cli/cmd/fs"
+	"github.com/databricks/cli/cmd/labs"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/cmd/sync"
 	"github.com/databricks/cli/cmd/version"
@@ -70,6 +71,7 @@ func New(ctx context.Context) *cobra.Command {
 	cli.AddCommand(bundle.New())
 	cli.AddCommand(configure.New())
 	cli.AddCommand(fs.New())
+	cli.AddCommand(labs.New(ctx))
 	cli.AddCommand(sync.New())
 	cli.AddCommand(version.New())
 

cmd/labs/CODEOWNERS

@@ -0,0 +1 @@
+* @nfx

cmd/labs/clear_cache.go

@@ -0,0 +1,33 @@
+package labs
+
+import (
+	"log/slog"
+	"os"
+
+	"github.com/databricks/cli/cmd/labs/project"
+	"github.com/databricks/cli/libs/log"
+	"github.com/spf13/cobra"
+)
+
+func newClearCacheCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "clear-cache",
+		Short: "Clears cache entries from everywhere relevant",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			projects, err := project.Installed(ctx)
+			if err != nil {
+				return err
+			}
+			_ = os.Remove(project.PathInLabs(ctx, "databrickslabs-repositories.json"))
+			logger := log.GetLogger(ctx)
+			for _, prj := range projects {
+				logger.Info("clearing labs project cache", slog.String("name", prj.Name))
+				_ = os.RemoveAll(prj.CacheDir(ctx))
+				// recreating empty cache folder for downstream apps to work normally
+				_ = prj.EnsureFoldersExist(ctx)
+			}
+			return nil
+		},
+	}
+}

cmd/labs/github/github.go

@@ -0,0 +1,66 @@
+package github
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/databricks/cli/libs/log"
+)
+
+const gitHubAPI = "https://api.github.com"
+const gitHubUserContent = "https://raw.githubusercontent.com"
+
+// Placeholders to use as unique keys in context.Context.
+var apiOverride int
+var userContentOverride int
+
+func WithApiOverride(ctx context.Context, override string) context.Context {
+	return context.WithValue(ctx, &apiOverride, override)
+}
+
+func WithUserContentOverride(ctx context.Context, override string) context.Context {
+	return context.WithValue(ctx, &userContentOverride, override)
+}
+
+var ErrNotFound = errors.New("not found")
+
+func getBytes(ctx context.Context, method, url string, body io.Reader) ([]byte, error) {
+	ao, ok := ctx.Value(&apiOverride).(string)
+	if ok {
+		url = strings.Replace(url, gitHubAPI, ao, 1)
+	}
+	uco, ok := ctx.Value(&userContentOverride).(string)
+	if ok {
+		url = strings.Replace(url, gitHubUserContent, uco, 1)
+	}
+	log.Tracef(ctx, "%s %s", method, url)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, body)
+	if err != nil {
+		return nil, err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode == 404 {
+		return nil, ErrNotFound
+	}
+	if res.StatusCode >= 400 {
+		return nil, fmt.Errorf("github request failed: %s", res.Status)
+	}
+	defer res.Body.Close()
+	return io.ReadAll(res.Body)
+}
+
+func httpGetAndUnmarshal(ctx context.Context, url string, response any) error {
+	raw, err := getBytes(ctx, "GET", url, nil)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(raw, response)
+}

cmd/labs/github/ref.go

@@ -0,0 +1,20 @@
+package github
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databricks/cli/libs/log"
+)
+
+func ReadFileFromRef(ctx context.Context, org, repo, ref, file string) ([]byte, error) {
+	log.Debugf(ctx, "Reading %s@%s from %s/%s", file, ref, org, repo)
+	url := fmt.Sprintf("%s/%s/%s/%s/%s", gitHubUserContent, org, repo, ref, file)
+	return getBytes(ctx, "GET", url, nil)
+}
+
+func DownloadZipball(ctx context.Context, org, repo, ref string) ([]byte, error) {
+	log.Debugf(ctx, "Downloading zipball for %s from %s/%s", ref, org, repo)
+	zipballURL := fmt.Sprintf("%s/repos/%s/%s/zipball/%s", gitHubAPI, org, repo, ref)
+	return getBytes(ctx, "GET", zipballURL, nil)
+}

cmd/labs/github/ref_test.go

@@ -0,0 +1,48 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFileFromRef(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/databrickslabs/ucx/main/README.md" {
+			w.Write([]byte(`abc`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithUserContentOverride(ctx, server.URL)
+
+	raw, err := ReadFileFromRef(ctx, "databrickslabs", "ucx", "main", "README.md")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("abc"), raw)
+}
+
+func TestDownloadZipball(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/repos/databrickslabs/ucx/zipball/main" {
+			w.Write([]byte(`abc`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	raw, err := DownloadZipball(ctx, "databrickslabs", "ucx", "main")
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("abc"), raw)
+}

cmd/labs/github/releases.go

@@ -0,0 +1,61 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/databricks/cli/cmd/labs/localcache"
+	"github.com/databricks/cli/libs/log"
+)
+
+const cacheTTL = 1 * time.Hour
+
+// NewReleaseCache creates a release cache for a repository in the GitHub org.
+// Caller has to provide different cache directories for different repositories.
+func NewReleaseCache(org, repo, cacheDir string) *ReleaseCache {
+	pattern := fmt.Sprintf("%s-%s-releases", org, repo)
+	return &ReleaseCache{
+		cache: localcache.NewLocalCache[Versions](cacheDir, pattern, cacheTTL),
+		Org:   org,
+		Repo:  repo,
+	}
+}
+
+type ReleaseCache struct {
+	cache localcache.LocalCache[Versions]
+	Org   string
+	Repo  string
+}
+
+func (r *ReleaseCache) Load(ctx context.Context) (Versions, error) {
+	return r.cache.Load(ctx, func() (Versions, error) {
+		return getVersions(ctx, r.Org, r.Repo)
+	})
+}
+
+// getVersions is considered to be a private API, as we want the usage go through a cache
+func getVersions(ctx context.Context, org, repo string) (Versions, error) {
+	var releases Versions
+	log.Debugf(ctx, "Fetching latest releases for %s/%s from GitHub API", org, repo)
+	url := fmt.Sprintf("%s/repos/%s/%s/releases", gitHubAPI, org, repo)
+	err := httpGetAndUnmarshal(ctx, url, &releases)
+	return releases, err
+}
+
+type ghAsset struct {
+	Name               string `json:"name"`
+	ContentType        string `json:"content_type"`
+	Size               int    `json:"size"`
+	BrowserDownloadURL string `json:"browser_download_url"`
+}
+
+type Release struct {
+	Version     string    `json:"tag_name"`
+	CreatedAt   time.Time `json:"created_at"`
+	PublishedAt time.Time `json:"published_at"`
+	ZipballURL  string    `json:"zipball_url"`
+	Assets      []ghAsset `json:"assets"`
+}
+
+type Versions []Release

cmd/labs/github/releases_test.go

@@ -0,0 +1,34 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadsReleasesForCLI(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/repos/databricks/cli/releases" {
+			w.Write([]byte(`[{"tag_name": "v1.2.3"}, {"tag_name": "v1.2.2"}]`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	r := NewReleaseCache("databricks", "cli", t.TempDir())
+	all, err := r.Load(ctx)
+	assert.NoError(t, err)
+	assert.Len(t, all, 2)
+
+	// no call is made
+	_, err = r.Load(ctx)
+	assert.NoError(t, err)
+}

cmd/labs/github/repositories.go

@@ -0,0 +1,59 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/databricks/cli/cmd/labs/localcache"
+	"github.com/databricks/cli/libs/log"
+)
+
+const repositoryCacheTTL = 24 * time.Hour
+
+func NewRepositoryCache(org, cacheDir string) *repositoryCache {
+	filename := fmt.Sprintf("%s-repositories", org)
+	return &repositoryCache{
+		cache: localcache.NewLocalCache[Repositories](cacheDir, filename, repositoryCacheTTL),
+		Org:   org,
+	}
+}
+
+type repositoryCache struct {
+	cache localcache.LocalCache[Repositories]
+	Org   string
+}
+
+func (r *repositoryCache) Load(ctx context.Context) (Repositories, error) {
+	return r.cache.Load(ctx, func() (Repositories, error) {
+		return getRepositories(ctx, r.Org)
+	})
+}
+
+// getRepositories is considered to be privata API, as we want the usage to go through a cache
+func getRepositories(ctx context.Context, org string) (Repositories, error) {
+	var repos Repositories
+	log.Debugf(ctx, "Loading repositories for %s from GitHub API", org)
+	url := fmt.Sprintf("%s/users/%s/repos", gitHubAPI, org)
+	err := httpGetAndUnmarshal(ctx, url, &repos)
+	return repos, err
+}
+
+type Repositories []ghRepo
+
+type ghRepo struct {
+	Name          string   `json:"name"`
+	Description   string   `json:"description"`
+	Langauge      string   `json:"language"`
+	DefaultBranch string   `json:"default_branch"`
+	Stars         int      `json:"stargazers_count"`
+	IsFork        bool     `json:"fork"`
+	IsArchived    bool     `json:"archived"`
+	Topics        []string `json:"topics"`
+	HtmlURL       string   `json:"html_url"`
+	CloneURL      string   `json:"clone_url"`
+	SshURL        string   `json:"ssh_url"`
+	License       struct {
+		Name string `json:"name"`
+	} `json:"license"`
+}

cmd/labs/github/repositories_test.go

@@ -0,0 +1,30 @@
+package github
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepositories(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/users/databrickslabs/repos" {
+			w.Write([]byte(`[{"name": "x"}]`))
+			return
+		}
+		t.Logf("Requested: %s", r.URL.Path)
+		panic("stub required")
+	}))
+	defer server.Close()
+
+	ctx := context.Background()
+	ctx = WithApiOverride(ctx, server.URL)
+
+	r := NewRepositoryCache("databrickslabs", t.TempDir())
+	all, err := r.Load(ctx)
+	assert.NoError(t, err)
+	assert.True(t, len(all) > 0)
+}