feat: responses saving in output folder (for -sr / -srd) and printing in console as json output (for -json)

Author: kartikeysemwal

cmd/cariddi/main.go

@@ -69,25 +69,26 @@ func main() {
 	// Setup the config according to the flags that were
 	// passed via the CLI
 	config := &crawler.Scan{
-		Delay:         flags.Delay,
-		Concurrency:   flags.Concurrency,
-		Ignore:        flags.Ignore,
-		IgnoreTxt:     flags.IgnoreTXT,
-		Cache:         flags.Cache,
-		JSON:          flags.JSON,
-		Timeout:       flags.Timeout,
-		Intensive:     flags.Intensive,
-		Rua:           flags.Rua,
-		Proxy:         flags.Proxy,
-		SecretsFlag:   flags.Secrets,
-		Plain:         flags.Plain,
-		EndpointsFlag: flags.Endpoints,
-		FileType:      flags.Extensions,
-		ErrorsFlag:    flags.Errors,
-		InfoFlag:      flags.Info,
-		Debug:         flags.Debug,
-		UserAgent:     flags.UserAgent,
-		StoreResp:     flags.StoreResp,
+		Delay:          flags.Delay,
+		Concurrency:    flags.Concurrency,
+		Ignore:         flags.Ignore,
+		IgnoreTxt:      flags.IgnoreTXT,
+		Cache:          flags.Cache,
+		JSON:           flags.JSON,
+		Timeout:        flags.Timeout,
+		Intensive:      flags.Intensive,
+		Rua:            flags.Rua,
+		Proxy:          flags.Proxy,
+		SecretsFlag:    flags.Secrets,
+		Plain:          flags.Plain,
+		EndpointsFlag:  flags.Endpoints,
+		FileType:       flags.Extensions,
+		ErrorsFlag:     flags.Errors,
+		InfoFlag:       flags.Info,
+		Debug:          flags.Debug,
+		UserAgent:      flags.UserAgent,
+		StoreResp:      flags.StoreResp,
+		StoredRespPath: flags.StoredRespDir,
 	}
 
 	// Read the targets from standard input.
@@ -118,18 +119,18 @@ func main() {
 	// Create output files if needed (txt / html).
 	config.Txt = ""
 	if flags.TXTout != "" {
-		config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt")
+		config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt", config.StoredRespPath)
 	}
 
 	var ResultHTML = ""
 	if flags.HTMLout != "" {
-		ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html")
+		ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html", config.StoredRespPath)
 		output.BannerHTML(ResultHTML)
 		output.HeaderHTML("Results", ResultHTML)
 	}
 
 	if config.StoreResp {
-		fileUtils.CreateIndexOutputFile("index.responses.txt")
+		fileUtils.CreateIndexOutputFile("index.responses.txt", config.StoredRespPath)
 	}
 
 	// Read headers if needed

internal/file/file.go

@@ -42,11 +42,16 @@ const (
 	Permission0644 = 0644
 )
 
+// constant defined in output.go as well, for circular dependency
+const (
+	CariddiOutputFolder = "output-cariddi"
+)
+
 // CreateOutputFolder creates the output folder
 // If it fails exits with an error message.
-func CreateOutputFolder() {
+func CreateOutputFolder(outputDir string) {
 	// Create a folder/directory at a full qualified path
-	err := os.Mkdir("output-cariddi", Permission0755)
+	err := os.MkdirAll(outputDir, Permission0755)
 	if err != nil {
 		fmt.Println("Can't create output folder.")
 		os.Exit(1)
@@ -56,9 +61,9 @@ func CreateOutputFolder() {
 // CreateHostOutputFolder creates the host output folder
 // for the HTTP responses.
 // If it fails exits with an error message.
-func CreateHostOutputFolder(host string) {
+func CreateHostOutputFolder(host string, outputDir string) {
 	// Create a folder/directory at a full qualified path
-	err := os.MkdirAll(filepath.Join("output-cariddi", host), Permission0755)
+	err := os.MkdirAll(filepath.Join(outputDir, host), Permission0755)
 	if err != nil {
 		fmt.Println("Can't create host output folder.")
 		os.Exit(1)
@@ -71,21 +76,25 @@ func CreateHostOutputFolder(host string) {
 // already exists, if yes asks the user if cariddi has to overwrite it;
 // if no cariddi creates it.
 // Whenever an instruction fails, it exits with an error message.
-func CreateOutputFile(target string, subcommand string, format string) string {
+func CreateOutputFile(target string, subcommand string, format string, outputDir string) string {
+	if outputDir == "" {
+		outputDir = CariddiOutputFolder
+	}
+
 	target = ReplaceBadCharacterOutput(target)
 
 	var filename string
 	if subcommand != "" {
-		filename = filepath.Join("output-cariddi", target+"."+subcommand+"."+format)
+		filename = filepath.Join(outputDir, target+"."+subcommand+"."+format)
 	} else {
-		filename = filepath.Join("output-cariddi", target+"."+format)
+		filename = filepath.Join(outputDir, target+"."+format)
 	}
 
 	_, err := os.Stat(filename)
 
 	if os.IsNotExist(err) {
-		if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
-			CreateOutputFolder()
+		if _, err := os.Stat(outputDir + "/"); os.IsNotExist(err) {
+			CreateOutputFolder(outputDir)
 		}
 		// If the file doesn't exist, create it.
 		f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
@@ -119,15 +128,19 @@ func CreateOutputFile(target string, subcommand string, format string) string {
 // It creates the output folder if needed, then checks if the index output file
 // already exists, if no cariddi creates it.
 // Whenever an instruction fails, it exits with an error message.
-func CreateIndexOutputFile(filename string) {
+func CreateIndexOutputFile(filename string, outputDir string) {
+	if outputDir == "" {
+		outputDir = CariddiOutputFolder
+	}
+
 	_, err := os.Stat(filename)
 
 	if os.IsNotExist(err) {
-		if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
-			CreateOutputFolder()
+		if _, err := os.Stat(outputDir + "/"); os.IsNotExist(err) {
+			CreateOutputFolder(outputDir)
 		}
 		// If the file doesn't exist, create it.
-		filename = filepath.Join("output-cariddi", filename)
+		filename = filepath.Join(outputDir, filename)
 
 		f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
 		if err != nil {

pkg/crawler/colly.go

@@ -134,8 +134,17 @@ func New(scan *Scan) *Results {
 			fmt.Println(r.Request.URL)
 		}
 
-		if scan.StoreResp {
-			err := output.StoreHTTPResponse(r)
+		var outputPath string
+
+		if scan.StoreResp || len(scan.StoredRespPath) > 0 {
+			outputDir := scan.StoredRespPath
+
+			if outputDir == "" {
+				outputDir = output.CariddiOutputFolder
+			}
+
+			var err error
+			outputPath, err = output.StoreHTTPResponse(r, outputDir)
 			if err != nil {
 				log.Println(err)
 			}
@@ -193,7 +202,7 @@ func New(scan *Scan) *Results {
 
 		if scan.JSON {
 			jsonOutput, err := output.GetJSONString(
-				r, secrets, parameters, filetype, errors, infos,
+				r, secrets, parameters, filetype, errors, infos, outputPath,
 			)
 
 			if err == nil {

pkg/crawler/options.go

@@ -39,26 +39,27 @@ type Results struct {
 
 type Scan struct {
 	// Flags
-	Cache         bool
-	Debug         bool
-	EndpointsFlag bool
-	ErrorsFlag    bool
-	InfoFlag      bool
-	Intensive     bool
-	Plain         bool
-	Rua           bool
-	SecretsFlag   bool
-	Ignore        string
-	IgnoreTxt     string
-	JSON          bool
-	HTML          string
-	Proxy         string
-	Target        string
-	Txt           string
-	UserAgent     string
-	FileType      int
-	Headers       map[string]string
-	StoreResp     bool
+	Cache          bool
+	Debug          bool
+	EndpointsFlag  bool
+	ErrorsFlag     bool
+	InfoFlag       bool
+	Intensive      bool
+	Plain          bool
+	Rua            bool
+	SecretsFlag    bool
+	Ignore         string
+	IgnoreTxt      string
+	JSON           bool
+	HTML           string
+	Proxy          string
+	Target         string
+	Txt            string
+	UserAgent      string
+	FileType       int
+	Headers        map[string]string
+	StoreResp      bool
+	StoredRespPath string
 
 	// Settings
 	Concurrency int

pkg/input/flags.go

@@ -94,6 +94,8 @@ type Input struct {
 	UserAgent string
 	// StoreResp stores HTTP responses.
 	StoreResp bool
+	// StoredRespDir stores HTTP responses to the directory provided.
+	StoredRespDir string
 }
 
 // ScanFlag defines all the options taken
@@ -142,6 +144,8 @@ func ScanFlag() Input {
 
 	storeRespPtr := flag.Bool("sr", false, "Store HTTP responses.")
 
+	storedRespDirPtr := flag.String("srd", "", "Stores HTTP responses to the directory provided.")
+
 	flag.Parse()
 
 	result := Input{
@@ -173,6 +177,7 @@ func ScanFlag() Input {
 		*debugPtr,
 		*userAgentPtr,
 		*storeRespPtr,
+		*storedRespDirPtr,
 	}
 
 	return result

pkg/output/jsonl.go

@@ -44,6 +44,7 @@ type JSONData struct {
 	ContentType   string          `json:"content_type,omitempty"`
 	ContentLength int             `json:"content_length,omitempty"`
 	Matches       *MatcherResults `json:"matches,omitempty"`
+	OutputPath    string          `json:"output_path,omitempty"`
 	// Host          string `json:"host"` # TODO: Available in Colly 2.x
 }
 
@@ -67,6 +68,7 @@ func GetJSONString(
 	filetype *scanner.FileType,
 	errors []scanner.ErrorMatched,
 	infos []scanner.InfoMatched,
+	outputPath string,
 ) ([]byte, error) {
 	// Parse response headers
 	headers := r.Headers
@@ -136,6 +138,7 @@ func GetJSONString(
 		ContentType:   contentType,
 		ContentLength: contentLength,
 		Matches:       matcherResults,
+		OutputPath:    outputPath,
 		// Host:          "", // TODO
 	}
 

pkg/output/jsonl_test.go

@@ -113,6 +113,7 @@ func TestJSONOutput(t *testing.T) {
 		errors     []scanner.ErrorMatched
 		infos      []scanner.InfoMatched
 		want       string
+		outputPath string
 	}{
 		{
 			name:       "test_all_findings",
@@ -123,6 +124,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     errors,
 			infos:      infos,
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_all_findings_nocontent",
@@ -133,6 +135,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     errors,
 			infos:      infos,
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_no_findings",
@@ -143,6 +146,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     []scanner.ErrorMatched{},
 			infos:      []scanner.InfoMatched{},
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128}`, //nolint: all
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_only_secrets",
@@ -153,6 +157,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     []scanner.ErrorMatched{},
 			infos:      []scanner.InfoMatched{},
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_only_params",
@@ -163,6 +168,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     []scanner.ErrorMatched{},
 			infos:      []scanner.InfoMatched{},
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_only_errors",
@@ -173,6 +179,7 @@ func TestJSONOutput(t *testing.T) {
 			errors:     errors,
 			infos:      []scanner.InfoMatched{},
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
 		},
 		{
 			name:       "test_only_infos",
@@ -183,12 +190,24 @@ func TestJSONOutput(t *testing.T) {
 			errors:     []scanner.ErrorMatched{},
 			infos:      infos,
 			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll
+			outputPath: "C:\\testDir1\\testDir2",
+		},
+		{
+			name:       "test_no_outputPath",
+			r:          resp,
+			secrets:    []scanner.SecretMatched{},
+			parameters: []scanner.Parameter{},
+			filetype:   &scanner.FileType{},
+			errors:     []scanner.ErrorMatched{},
+			infos:      infos,
+			want:       `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll
+			outputPath: "",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { //nolint:lll
+			if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos, tt.outputPath); !reflect.DeepEqual(string(got), tt.want) { //nolint:lll
 				t.Errorf("GetJSONString\n%v", string(got))
 				t.Errorf("want\n%v", tt.want)
 			}