Support output formats

Adds support for printing CSV, JSON, Markdown, and LaTeX. Includes
support for new --output-format CLI argument as well as new system
functions )output.* for toggling output formats interactively at the
REPL.

Author: semperos

README.md

@@ -50,19 +50,19 @@ Non-exhaustive list:
 - TODO: Test coverage.
 - TODO: Correct usage of `goal.NewError` vs. `goal.NewPanicError`
 - TODO: Option for raw REPL (modeled on Goal's) with better input performance, no auto-complete etc.
-- TODO: Goal function to return auto-complete results (esp. if raw REPL is being used).
-- TODO: Looser auto-complete, not just prefix-based
 - TODO: Functions to conveniently populate SQL tables with Goal values.
 - TODO: Support plots/charts (consider https://github.com/wcharczuk/go-chart)
 - TODO: User commands (as found in [APL](https://aplwiki.com/wiki/User_command)), executable from Goal or SQL modes
 
 I plan to support the above items. The following are stretch goals or nice-to-have's:
 
+- TODO: Use custom table functions via replacement scan to query Goal tables from DuckDB.
+- TODO: Looser auto-complete, not just prefix-based
+- TODO: `)help`
 - TODO: Functions leveraging [time.Time](https://pkg.go.dev/time@go1.22.5)
-- TODO: `tui.` functions in CLI mode using https://github.com/charmbracelet/lipgloss (already a transitive dependency) for colored output, etc.
+- IN PROGRESS: `tui.` functions in CLI mode using https://github.com/charmbracelet/lipgloss (already a transitive dependency) for colored output, etc.
 - TODO: Implement a subset of [q](https://code.kx.com/q/) functions to extend what Goal already has.
 - Specific user commands:
-  - TODO: Choosing output format (e.g., as JSON, perhaps all the ones DuckDB supports)
   - TODO: Toggle pretty-printing
   - TODO: Toggle paging at the REPL (as found in [PicoLisp](https://picolisp.com/wiki/?home))
   - TODO: Toggle colored output

cmd/ari/input.go

@@ -397,7 +397,18 @@ func (autoCompleter *AutoCompleter) cacheSystemCommands() {
 
 func systemCommands() (map[string]string, []string) {
 	m := map[string]string{
-		")goal": "Goal array language mode", ")sql": "Read-only SQL mode (querying)", ")sql!": "Read/write SQL mode",
+		")goal": "Goal array language mode",
+		// TODO Output formats: https://duckdb.org/docs/api/cli/output_formats.html
+		// In particular csv, json, markdown, latex, and one of the boxed ones
+		")output.csv":         "Print results as CSV",
+		")output.goal":        "Print results as Goal values (default)",
+		")output.json":        "Print results as JSON",
+		")output.json+pretty": "Print results as JSON with indentation",
+		")output.latex":       "Print results as LaTeX",
+		")output.markdown":    "Print results as Markdown",
+		")output.tsv":         "Print results as TSV",
+		")sql":                "Read-only SQL mode (querying)",
+		")sql!":               "Read/write SQL mode",
 	}
 	// Prepare sorted keys ahead of time
 	keys := make([]string, 0, len(m))

cmd/ari/root.go

@@ -33,6 +33,18 @@ const (
 	cliModeSQLReadWrite
 )
 
+type outputFormat int
+
+const (
+	outputFormatGoal outputFormat = iota
+	outputFormatCSV
+	outputFormatJSON
+	outputFormatJSONPretty
+	outputFormatLatex
+	outputFormatMarkdown
+	outputFormatTSV
+)
+
 const (
 	cliModeGoalPrompt             = "  "
 	cliModeGoalNextPrompt         = "  "
@@ -43,11 +55,12 @@ const (
 )
 
 type CliSystem struct {
+	ariContext    *ari.Context
+	autoCompleter *AutoCompleter
 	cliEditor     *bubbline.Editor
 	cliMode       cliMode
-	autoCompleter *AutoCompleter
-	ariContext    *ari.Context
 	debug         bool
+	outputFormat  outputFormat
 	programName   string
 }
 
@@ -138,7 +151,28 @@ func cliModeFromString(s string) (cliMode, error) {
 	case "sql!":
 		return cliModeSQLReadWrite, nil
 	default:
-		return 0, errors.New("unsupported ari mode: " + s)
+		return 0, errors.New("unsupported --mode: " + s)
+	}
+}
+
+func outputFormatFromString(s string) (outputFormat, error) {
+	switch s {
+	case "csv":
+		return outputFormatCSV, nil
+	case "goal":
+		return outputFormatGoal, nil
+	case "json":
+		return outputFormatJSON, nil
+	case "json+pretty":
+		return outputFormatJSONPretty, nil
+	case "latex":
+		return outputFormatLatex, nil
+	case "markdown":
+		return outputFormatMarkdown, nil
+	case "tsv":
+		return outputFormatTSV, nil
+	default:
+		return 0, errors.New("unsupported --output-format: " + s)
 	}
 }
 
@@ -191,16 +225,27 @@ func ariMain(cmd *cobra.Command, args []string) int {
 		defer pprof.StopCPUProfile()
 	}
 
+	// Defaults to outputFormatGoal
+	startupOutputFormatString := viper.GetString("output-format")
+	startupOutputFormat, err := outputFormatFromString(startupOutputFormatString)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		return 1
+	}
+	mainCliSystem.outputFormat = startupOutputFormat
+
 	// MUST PRECEDE EXECUTE/REPL
 	goalFilesToLoad := viper.GetStringSlice("load")
 	for _, f := range goalFilesToLoad {
-		err = runScript(&mainCliSystem, f)
+		_, err = runScript(&mainCliSystem, f)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Failed to load file %q with error: %v", f, err)
 			return 1
 		}
 	}
 
+	// By default, we don't print the final return value of a script, but this flag supports that.
+	printFinalValue := viper.GetBool("println")
 	// Support file argument both with -e and standalone.
 	hasFileArgument := len(args) > 0
 
@@ -211,13 +256,16 @@ func ariMain(cmd *cobra.Command, args []string) int {
 		return 1
 	}
 	if programToExecute != "" {
-		err = runCommand(&mainCliSystem, programToExecute)
-		if err != nil {
+		goalV, errr := runCommand(&mainCliSystem, programToExecute)
+		if errr != nil {
 			fmt.Fprintf(os.Stderr, "Failed to execute program:\n%q\n    with error:\n%v\n", programToExecute, err)
 			return 1
 		}
 		// Support -e/--execute along with a file argument.
 		if !hasFileArgument {
+			if printFinalValue {
+				printInOutputFormat(ariContext.GoalContext, mainCliSystem.outputFormat, goalV)
+			}
 			return 0
 		}
 	}
@@ -230,11 +278,14 @@ func ariMain(cmd *cobra.Command, args []string) int {
 			fmt.Fprintf(os.Stderr, "File %q is not recognized as a path on your system: %v", f, err)
 		}
 		ariContext.GoalContext.AssignGlobal("FILE", goal.NewS(path))
-		err = runScript(&mainCliSystem, f)
-		if err != nil {
+		goalV, errr := runScript(&mainCliSystem, f)
+		if errr != nil {
 			fmt.Fprintf(os.Stderr, "Failed to run file %q with error: %v", f, err)
 			return 1
 		}
+		if printFinalValue {
+			printInOutputFormat(ariContext.GoalContext, mainCliSystem.outputFormat, goalV)
+		}
 		return 0
 	}
 
@@ -332,18 +383,54 @@ func (cliSystem *CliSystem) replEvalGoal(line string) {
 	}
 
 	if !goalContext.AssignedLast() {
-		ariPrintFn := cliSystem.detectAriPrint()
+		// In the REPL, make it easy to get the value of the _p_revious expression
+		// just evaluated. Equivalent of *1 in Lisp REPLs. Skip assignments.
+		printInOutputFormat(goalContext, cliSystem.outputFormat, value)
+	}
+
+	cliSystem.detectAriPrompt()
+}
+
+func printInOutputFormat(goalContext *goal.Context, outputFormat outputFormat, value goal.V) {
+	goalContext.AssignGlobal("ari.p", value)
+	switch outputFormat {
+	case outputFormatGoal:
+		ariPrintFn := detectAriPrint(goalContext)
 		if ariPrintFn != nil {
 			ariPrintFn(value)
 		} else {
 			fmt.Fprintln(os.Stdout, value.Sprint(goalContext, false))
 		}
-		// In the REPL, make it easy to get the value of the _p_revious expression
-		// just evaluated. Equivalent of *1 in Lisp REPLs. Skip assignments.
-		goalContext.AssignGlobal("ari.p", value)
+	case outputFormatCSV:
+		evalThen(goalContext, value, `csv ari.p`)
+	case outputFormatJSON:
+		evalThen(goalContext, value, `""json ari.p`)
+	case outputFormatJSONPretty:
+		evalThen(goalContext, value, `" "json ari.p`)
+	case outputFormatLatex:
+		evalThen(goalContext, value, `out.ltx[ari.p;"%.2f"]`)
+	case outputFormatMarkdown:
+		evalThen(goalContext, value, `out.md[ari.p;"%.2f"]`)
+	case outputFormatTSV:
+		evalThen(goalContext, value, `"\t"csv ari.p`)
 	}
+}
 
-	cliSystem.detectAriPrompt()
+// evalThen evaluates the given goalProgram for side effects, with ari.p already bound to previous evaluation.
+func evalThen(goalContext *goal.Context, value goal.V, goalProgram string) {
+	nextValue, err := goalContext.Eval(goalProgram)
+	if err != nil {
+		formatREPLError(err)
+	}
+	if value.IsError() {
+		formatREPLError(newExitError(goalContext, value.Error()))
+	}
+	switch jsonS := nextValue.BV().(type) {
+	case goal.S:
+		fmt.Fprintln(os.Stdout, string(jsonS))
+	default:
+		formatREPLError(errors.New("developer error: json must produce a string"))
+	}
 }
 
 // ExitError is returned by Cmd when the program returns a Goal error value.
@@ -411,8 +498,7 @@ func (cliSystem *CliSystem) detectAriPrompt() {
 }
 
 // detectAriPrint returns a function for printing values at the REPL in goal mode.
-func (cliSystem *CliSystem) detectAriPrint() func(goal.V) {
-	goalContext := cliSystem.ariContext.GoalContext
+func detectAriPrint(goalContext *goal.Context) func(goal.V) {
 	printFn, found := goalContext.GetGlobal("ari.print")
 	if found {
 		if printFn.IsCallable() {
@@ -459,9 +545,22 @@ func (cliSystem *CliSystem) replEvalSystemCommand(line string) error {
 	cmdAndArgs := strings.Split(line, " ")
 	systemCommand := cmdAndArgs[0]
 	switch systemCommand {
-	// IDEA )help that doesn't require quoting
 	case ")goal":
 		return cliSystem.switchMode(cliModeGoal, nil)
+	case ")output.goal":
+		cliSystem.outputFormat = outputFormatGoal
+	case ")output.csv":
+		cliSystem.outputFormat = outputFormatCSV
+	case ")output.json":
+		cliSystem.outputFormat = outputFormatJSON
+	case ")output.json+pretty":
+		cliSystem.outputFormat = outputFormatJSONPretty
+	case ")output.latex":
+		cliSystem.outputFormat = outputFormatLatex
+	case ")output.markdown":
+		cliSystem.outputFormat = outputFormatMarkdown
+	case ")output.tsv":
+		cliSystem.outputFormat = outputFormatTSV
 	case ")sql":
 		return cliSystem.switchMode(cliModeSQLReadOnly, cmdAndArgs[1:])
 	case ")sql!":
@@ -510,43 +609,43 @@ func debugPrintStack(ctx *goal.Context, programName string) {
 }
 
 // Adapted from Goal's implementation.
-func runCommand(cliSystem *CliSystem, cmd string) error {
+func runCommand(cliSystem *CliSystem, cmd string) (goal.V, error) {
 	return runSource(cliSystem, cmd, "")
 }
 
 // Adapted from Goal's implementation.
-func runScript(cliSystem *CliSystem, fname string) error {
+func runScript(cliSystem *CliSystem, fname string) (goal.V, error) {
 	bs, err := os.ReadFile(fname)
 	if err != nil {
-		return fmt.Errorf("%s: %w", cliSystem.programName, err)
+		return goal.NewGap(), fmt.Errorf("%s: %w", cliSystem.programName, err)
 	}
 	// We avoid redundant copy in bytes->string conversion.
 	source := unsafe.String(unsafe.SliceData(bs), len(bs))
 	return runSource(cliSystem, source, fname)
 }
 
 // Adapted from Goal's implementation.
-func runSource(cliSystem *CliSystem, source, loc string) error {
+func runSource(cliSystem *CliSystem, source, loc string) (goal.V, error) {
 	goalContext := cliSystem.ariContext.GoalContext
 	err := goalContext.Compile(source, loc, "")
 	if err != nil {
 		if cliSystem.debug {
 			printProgram(goalContext, cliSystem.programName)
 		}
-		return formatError(cliSystem.programName, err)
+		return goal.NewGap(), formatError(cliSystem.programName, err)
 	}
 	if cliSystem.debug {
 		printProgram(goalContext, cliSystem.programName)
-		return nil
+		return goal.NewGap(), nil
 	}
 	r, err := goalContext.Run()
 	if err != nil {
-		return formatError(cliSystem.programName, err)
+		return r, formatError(cliSystem.programName, err)
 	}
 	if r.IsError() {
-		return fmt.Errorf("%s", formatGoalError(goalContext, r))
+		return r, fmt.Errorf("%s", formatGoalError(goalContext, r))
 	}
-	return nil
+	return r, nil
 }
 
 // printProgram prints debug information about the context and any compiled
@@ -655,21 +754,15 @@ working with SQL and HTTP APIs.`,
 	var cfgFile string
 	cobra.OnInitialize(initConfigFn(cfgFile))
 
-	// Here you will define your flags and configuration settings.
-	// Cobra supports persistent flags, which, if defined here,
-	// will be global for your application.
-
 	home, err := os.UserHomeDir()
 	cobra.CheckErr(err)
 	cfgDir := path.Join(home, ".config", "ari")
-
 	defaultHistFile := path.Join(cfgDir, "ari-history.txt")
 	defaultCfgFile := path.Join(cfgDir, "ari-config.yaml")
 
 	// Config file has processing in initConfigFn outside of viper lifecycle, so it's a separate variable.
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", defaultCfgFile, "ari configuration")
 
-	// Everything else should go through viper for consistency.
 	pFlags := rootCmd.PersistentFlags()
 
 	flagNameHistory := "history"
@@ -692,6 +785,12 @@ working with SQL and HTTP APIs.`,
 	rootCmd.Flags().StringP("mode", "m", "goal", "language mode at startup")
 	err = viper.BindPFlag("mode", rootCmd.Flags().Lookup("mode"))
 	cobra.CheckErr(err)
+	rootCmd.Flags().StringP("output-format", "f", "goal", "evaluation output format")
+	err = viper.BindPFlag("output-format", rootCmd.Flags().Lookup("output-format"))
+	cobra.CheckErr(err)
+	rootCmd.Flags().BoolP("println", "p", false, "print final value of the script + newline")
+	err = viper.BindPFlag("println", rootCmd.Flags().Lookup("println"))
+	cobra.CheckErr(err)
 	rootCmd.Flags().BoolP("version", "v", false, "print version info and exit")
 
 	// NB: MUST be last in this method.

goal.go

@@ -23,15 +23,24 @@ const (
 // goalLoadExtendedPreamble loads the goalSource* snippets below,
 // loading them into the Goal context.
 func goalLoadExtendedPreamble(ctx *goal.Context) error {
-	goalPackages := map[string]string{
-		"":     goalSourceShape + goalSourceTable,
+	corePackages := map[string]string{
+		"": goalSourceShape + "\n" + goalSourceTable,
+	}
+	additionalPackages := map[string]string{
 		"fmt":  goalSourceFmt,
 		"html": goalSourceHTML,
 		"k":    goalSourceK,
+		"out":  goalSourceOut,
 		"math": goalSourceMath,
 		"mods": goalSourceMods,
 	}
-	for pkg, source := range goalPackages {
+	for pkg, source := range corePackages {
+		_, err := ctx.EvalPackage(source, "<builtin>", pkg)
+		if err != nil {
+			return err
+		}
+	}
+	for pkg, source := range additionalPackages {
 		_, err := ctx.EvalPackage(source, "<builtin>", pkg)
 		if err != nil {
 			return err
@@ -58,6 +67,9 @@ var goalSourceMods string
 //go:embed vendor-goal/shape.goal
 var goalSourceShape string
 
+//go:embed vendor-goal/out.goal
+var goalSourceOut string
+
 //go:embed vendor-goal/table.goal
 var goalSourceTable string
 

testing/table.goal

@@ -1,3 +1,2 @@
 t:csv.t ","csv 'read"data/starwars.csv";(#*t;!t) / (87;"name""height""mass""hair_color""skin_color""eye_color""birth_year""sex""gender""homeworld""species""films""vehicles""starships")
 t:json.t@json rq/[{"a":1,"b":2},{"a":10,"b":20},{"a":100,"b":200}]/;(#*t;!t) / (3;"a""b")
-