Centralize the configuration of the OSDF client.

.goreleaser.yml

@@ -128,9 +128,8 @@ nfpms:
           - src: resources/10-stash-plugin.conf
             dst: "/etc/condor/config.d/10-stash-plugin.conf"
             type: config|noreplace
-        replacements:
-          amd64: x86_64
-        file_name_template: "{{ .PackageName }}-{{ .Version }}-{{ .Release }}.{{ .Arch }}"
+        file_name_template: >-
+          {{ .PackageName }}-{{ .Version }}-{{ .Release }}.{{ if eq .Arch "amd64" }}x86_64{{ else }}{{ .Arch }}{{ end }}
       deb:
         contents:
           - src: LICENSE.txt
@@ -170,11 +169,10 @@ nfpms:
             dst: "/usr/share/doc/{{ .PackageName }}-{{ .Version }}/LICENSE.txt"
           - src: README.md
             dst: "/usr/share/doc/{{ .PackageName }}-{{ .Version }}/README.md"
-        replacements:
-          amd64: x86_64
         replaces:
           - stashcache-client < 6.4.0
-        file_name_template: "{{ .PackageName }}-{{ .Version }}-{{ .Release }}.{{ .Arch }}"
+        file_name_template: >-
+          {{ .PackageName }}-{{ .Version }}-{{ .Release }}.{{ if eq .Arch "amd64" }}x86_64{{ else }}{{ .Arch }}{{ end }}
       deb:
         file_name_template: "{{ .PackageName }}-{{ .Version }}-{{ .Release }}_{{ .Arch }}"
         contents:

cmd/stash_plugin/main.go

@@ -10,6 +10,7 @@ import (
 
 	stashcp "github.com/htcondor/osdf-client/v6"
 	"github.com/htcondor/osdf-client/v6/classads"
+	"github.com/htcondor/osdf-client/v6/config"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -26,8 +27,13 @@ type Transfer struct {
 }
 
 func main() {
-	// Parse command line arguments
+	err := config.Init()
+	if err != nil {
+		log.Errorln(err)
+		os.Exit(1)
+	}
 
+	// Parse command line arguments
 	var upload bool = false
 	// Set the options
 	stashcp.Options.Recursive = false

cmd/stashcp/main.go

@@ -2,12 +2,12 @@ package main
 
 import (
 	"fmt"
-	"net/url"
 	"os"
 	"strings"
 
 	stashcp "github.com/htcondor/osdf-client/v6"
 	namespaces "github.com/htcondor/osdf-client/v6/namespaces"
+	"github.com/htcondor/osdf-client/v6/config"
 	"github.com/jessevdk/go-flags"
 	log "github.com/sirupsen/logrus"
 )
@@ -79,7 +79,13 @@ var parser = flags.NewParser(&options, flags.Default)
 func main() {
 
 	stashcp.Options.Version = version
-
+
+	err := config.Init()
+	if err != nil {
+		log.Errorln(err)
+		os.Exit(1)
+	}
+
 	// Capture the start time of the transfer
 	if _, err := parser.Parse(); err != nil {
 		if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp {
@@ -168,34 +174,10 @@ func main() {
 	log.Debugln("Sources:", source)
 	log.Debugln("Destination:", dest)
 	if options.ListDir {
-		dirUrl, _ := url.Parse("http://stash.osgconnect.net:1094")
-		dirUrl.Path = source[0]
-		isDir, err := stashcp.IsDir(dirUrl, "", namespaces.Namespace{})
-		if err != nil {
-			log.Errorln("Error getting directory listing:", err)
-		}
-		log.Debugln("Dir is a directory?", isDir)
-		return
+		log.Errorln("Directory listing not currently implemented")
+		os.Exit(1)
 	}
 
-	/*
-		TODO: Parse a caches JSON, is this needed anymore?
-		if args.caches_json {
-			caches_json_location = caches_json
-
-		} else if val, jsonPresent := os.LookupEnv("CACHES_JSON"); jsonPresent {
-			caches_json_location = val
-		} else {
-			prefix = os.Getenv("OSG_LOCATION", "/")
-			caches_file = filepath.Join(prefix, "etc/stashcache/caches.json")
-			if _, err := os.Stat(caches_file); err == nil {
-				caches_json_location = caches_file
-			}
-		}
-
-		caches_list_name = args.cache_list_name
-	*/
-
 	// Check for manually entered cache to use ??
 	nearestCache, nearestCacheIsPresent := os.LookupEnv("NEAREST_CACHE")
 

config/config.go

@@ -1,6 +1,15 @@
 
 package config
 
+import (
+	"path/filepath"
+	"strconv"
+	"strings"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
 
 // Structs holding the OAuth2 state (and any other OSDF config needed)
 
@@ -26,3 +35,128 @@ type OSDFConfig struct {
 		OauthClient [] PrefixEntry `yaml:"oauth_client,omitempty"`
 	} `yaml:"OSDF"`
 }
+
+//
+// Based on the name of the current binary, determine the preferred "style"
+// of behavior.  For example, a binary with the "osdf_" prefix should utilize
+// the known URLs for OSDF.  For "pelican"-style commands, the user will
+// need to manually configure the location of the director endpoint.
+//
+func GetPreferredPrefix() string {
+	arg0 := strings.ToUpper(filepath.Base(os.Args[0]))
+	underscore_idx := strings.Index(arg0, "_")
+	if underscore_idx != -1 {
+		return arg0[0:underscore_idx]
+	}
+	if strings.HasPrefix(arg0, "STASH") {
+		return "STASH"
+	} else if strings.HasPrefix(arg0, "OSDF") {
+		return "OSDF"
+	}
+	return "PELICAN"
+}
+
+//
+// Get the list of valid prefixes for this binary.  Given there's been so
+// many renames of the project (stash -> osdf -> pelican), we allow multiple
+// prefixes when searching through environment variables.
+//
+func GetAllPrefixes() []string {
+	prefixes := []string{GetPreferredPrefix()}
+
+	if prefixes[0] == "OSDF" {
+		prefixes = append(prefixes, "STASH", "PELICAN")
+	} else if prefixes[0] == "STASH" {
+		prefixes = append(prefixes, "OSDF", "PELICAN")
+	}
+	return prefixes
+}
+
+func Init() error {
+	upper_prefix := GetPreferredPrefix()
+	lower_prefix := strings.ToLower(upper_prefix)
+
+	viper.SetDefault("StoppedTransferTimeout", 100)
+	viper.SetDefault("SlowTransferRampupTime", 100)
+	viper.SetDefault("SlowTransferWindow", 30)
+
+	if upper_prefix == "OSDF" || upper_prefix == "STASH" {
+		viper.SetDefault("NamespaceURL", "https://topology.opensciencegrid.org/osdf/namespaces")
+	}
+
+	viper.SetEnvPrefix(upper_prefix)
+	viper.AutomaticEnv()
+
+	viper.SetConfigName("config")
+	viper.SetConfigType("yaml")
+	viper.AddConfigPath("$HOME/." + lower_prefix)
+	err := viper.ReadInConfig()
+	if err != nil {
+		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+			return err
+		}
+		// Do not fail if the config file is missing
+	}
+	env_config_file := os.Getenv(upper_prefix + "_CONFIG_FILE")
+	if len(env_config_file) != 0 {
+		fp, err := os.Open(env_config_file)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		err = viper.ReadConfig(fp)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Handle all the grandfathered configuration parameters
+	prefixes := GetAllPrefixes()
+	prefixes_with_osg := append(prefixes, "OSG")
+	for _, prefix := range prefixes_with_osg {
+		if _, isSet := os.LookupEnv(prefix + "_DISABLE_HTTP_PROXY"); isSet {
+			viper.Set("DisableHttpProxy", true)
+			break
+		}
+	}
+	for _, prefix := range prefixes_with_osg {
+		if _, isSet := os.LookupEnv(prefix + "_DISABLE_PROXY_FALLBACK"); isSet {
+			viper.Set("DisableProxyFallback", true)
+			break
+		}
+	}
+	for _, prefix := range prefixes {
+		if val, isSet := os.LookupEnv(prefix + "_DIRECTOR_URL"); isSet {
+			viper.Set("DirectorURL", val)
+			break
+		}
+	}
+	for _, prefix := range prefixes {
+		if val, isSet := os.LookupEnv(prefix + "_NAMESPACE_URL"); isSet {
+			viper.Set("NamespaceURL", val)
+			break
+		}
+	}
+
+	// Check the environment variable STASHCP_MINIMUM_DOWNLOAD_SPEED (and all the prefix variants)
+	var downloadLimit int64 = 1024 * 100
+	var prefixes_with_cp []string
+	for _, prefix := range prefixes {
+		prefixes_with_cp = append(prefixes_with_cp, prefix + "CP")
+	}
+	for _, prefix := range append(prefixes, prefixes_with_cp...) {
+		downloadLimitStr := os.Getenv(prefix + "_MINIMUM_DOWNLOAD_SPEED")
+		if len(downloadLimitStr) == 0 {
+			continue
+		}
+		var err error
+		downloadLimit, err = strconv.ParseInt(downloadLimitStr, 10, 64)
+		if err != nil {
+			log.Errorf("Environment variable %s_MINIMUM_DOWNLOAD_SPEED=%s is not parsable as integer: %s",
+			    prefixes, downloadLimitStr, err.Error())
+		}
+		break
+	}
+	viper.Set("MinimumDownloadSpeed", downloadLimit)
+
+	return nil
+}

errorAccum_test.go

@@ -21,7 +21,7 @@ func TestErrorAccum(t *testing.T) {
 	AddError(err2)
 
 	errStr := GetErrors()
-	assert.Equal(t, "Attempt #2: error2 (0s elapsed, 0s since start); Attempt #1: error1 (0s since start)", errStr)
+	assert.Regexp(t, `Attempt\ \#2:\ error2\ \(0s\ elapsed,\ [0-9]+m?s\ since\ start\);\ Attempt\ \#1:\ error1\ \([0-9]+m?s\ since\ start\)`, errStr)
 
 }
 

go.mod

@@ -9,13 +9,14 @@ require (
 	github.com/jsipprell/keyctl v1.0.4-0.20211208153515-36ca02672b6c
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.2.2
+	github.com/spf13/viper v1.16.0
+	github.com/stretchr/testify v1.8.3
 	github.com/studio-b12/gowebdav v0.0.0-20210203212356-8244b5a5f51a
 	github.com/vbauerster/mpb/v7 v7.1.5
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.2.0
-	golang.org/x/oauth2 v0.2.0
-	golang.org/x/term v0.2.0
+	golang.org/x/crypto v0.9.0
+	golang.org/x/net v0.10.0
+	golang.org/x/oauth2 v0.7.0
+	golang.org/x/term v0.8.0
 	gopkg.in/yaml.v3 v3.0.1
 )