add check of domain age to avoid flood

Signed-off-by: Issif <issif_github@gadz.org>

Author: Issif

README.md

@@ -27,12 +27,17 @@ Two methods are available for configuration and can be mixed :
 
 ### With config file
 
+The available settings are:
 ```bash
 ---
-SlackWebhookURL: "" #Slack Webhook URL
-SlackIconURL: "" #Slack Icon (Avatar) URL
-SlackUsername: "" #Slack Username
-Regexp: ".*\\.fr$" #Regexp to match. Can't be empty. It uses Golang regexp format
+SlackWebhookURL: "" # Slack Webhook URL
+SlackIconURL: "" # Slack Icon (Avatar) URL
+SlackUsername: "Cercat" # Slack Username
+Regexp: ".*\\.fr$" # Regexp to match. Can't be empty. It uses Golang regexp format
+Workers: 20 # Number of workers to compare the certificates with the Regexp
+TakeScreenshot: false # Try to take a screenshot of the website
+ScreenshotsFolder: "." # Folder to store the screenshots
+IgnoreOlderThan: 10 # Ignore domains older than
 ```
 
 ### With env vars
@@ -41,6 +46,10 @@ Regexp: ".*\\.fr$" #Regexp to match. Can't be empty. It uses Golang regexp forma
 - **SLACKICONURL**: Slack Icon (Avatar) URL
 - **SLACKUSERNAME**: Slack Username
 - **REGEXP**: Regexp to match. Can't be empty. It uses Golang regexp format
+- **WORKERS**: Number of workers to compare the certificates with the Regexp
+- **TAKESCREENSHOT**: Try to take a screenshot of the website
+- **SCREENSHOTSFOLDER**: Folder to store the screenshots
+- **IGNOREOLDERTHAN**: Ignore domains older than
 
 ## Run
 

cmd/main.go

@@ -6,7 +6,6 @@ import (
 	"cercat/pkg/model"
 	"cercat/pkg/slack"
 	"cercat/pkg/worker"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,7 +15,6 @@ import (
 
 	"github.com/arl/statsviz"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -38,6 +36,7 @@ func main() {
 	}
 
 	cfg := config.CreateConfig(configFile)
+	// fmt.Printf("%#v\n", cfg)
 	for i := 0; i < cfg.Workers; i++ {
 		go worker.RunCertCheckWorker(cfg)
 	}
@@ -58,8 +57,8 @@ func runNotifierWorker(cfg *config.Configuration) {
 		if !duplicate {
 			cfg.PreviousCerts = cfg.PreviousCerts.Prev()
 			cfg.PreviousCerts.Value = result.Domain
-			j, _ := json.Marshal(result)
-			log.Infof("A certificate for '%v' has been issued : %v\n", result.Domain, string(j))
+			// j, _ := json.Marshal(result)
+			cfg.Log.Infof("A certificate for '%v' has been issued by %v\n", result.Domain, result.Registrar)
 			if cfg.SlackWebHookURL != "" {
 				go func(c *config.Configuration, r *model.Result) {
 					slack.NewPayload(c, result).Post(c)

config/config.go

@@ -4,11 +4,12 @@ import (
 	"cercat/pkg/homoglyph"
 	"cercat/pkg/model"
 	"container/ring"
+	"os"
 	"path"
 	"path/filepath"
 	"regexp"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
 
@@ -18,14 +19,17 @@ type Configuration struct {
 	SlackWebHookURL   string
 	SlackIconURL      string
 	SlackUsername     string
-	RegIP             string
 	Regexp            string
+	ScreenshotsFolder string
+	TakeScreenshot    bool
+	IgnoreOlderThan   int
+	RegIP             string
 	RegexpC           *regexp.Regexp
 	PreviousCerts     *ring.Ring
 	Messages          chan []byte
 	Buffer            chan *model.Result
 	Homoglyph         map[string]string
-	ScreenshotsFolder string
+	Log               *logrus.Logger
 }
 
 // CreateConfig provides a Configuration
@@ -34,16 +38,26 @@ func CreateConfig(configFile *string) *Configuration {
 		Workers:       50,
 		Homoglyph:     homoglyph.GetHomoglyphMap(),
 		PreviousCerts: ring.New(20),
-		Messages:      make(chan []byte, 50),
-		Buffer:        make(chan *model.Result, 50),
+		Messages:      make(chan []byte, 100),
+		Buffer:        make(chan *model.Result, 100),
+		Log:           logrus.New(),
 	}
 
+	c.Log.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp:   true,
+		TimestampFormat: "2006-01-02 15:04:05",
+	})
+	c.Log.SetOutput(os.Stdout)
+
 	v := viper.New()
 	v.SetDefault("SlackWebhookURL", "")
 	v.SetDefault("SlackIconURL", "")
 	v.SetDefault("SlackUsername", "Cercat")
 	v.SetDefault("Regexp", "")
 	v.SetDefault("Workers", 20)
+	v.SetDefault("TakeScreenshot", false)
+	v.SetDefault("ScreenshotsFolder", ".")
+	v.SetDefault("IgnoreOlderThan", 10)
 
 	if *configFile != "" {
 		d, f := path.Split(*configFile)
@@ -54,7 +68,7 @@ func CreateConfig(configFile *string) *Configuration {
 		v.AddConfigPath(d)
 		err := v.ReadInConfig()
 		if err != nil {
-			log.Fatalf("[ERROR] : Error when reading config file : %v\n", err)
+			c.Log.Fatalf("[ERROR] : Error when reading config file : %v\n", err)
 		}
 	}
 	v.AutomaticEnv()
@@ -65,14 +79,13 @@ func CreateConfig(configFile *string) *Configuration {
 	}
 
 	if c.Regexp == "" {
-		log.Fatal("Regexp can't be empty")
+		c.Log.Fatal("Regexp can't be empty")
 	}
 
 	reg, err := regexp.Compile(c.Regexp)
 	if err != nil {
-		log.Fatal("Bad regexp")
+		c.Log.Fatal("Bad regexp")
 	}
 	c.RegexpC = reg
-
 	return c
 }

go.mod

@@ -20,6 +20,7 @@ require (
 )
 
 require (
+	github.com/jpillora/go-tld v1.2.1
 	github.com/likexian/whois v1.14.4
 	github.com/likexian/whois-parser v1.24.2
 )
@@ -33,7 +34,6 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jpillora/go-tld v1.2.1 // indirect
 	github.com/likexian/gokit v0.25.9 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

lib/lib_suite_test.go

@@ -3,11 +3,12 @@ package certstream
 import (
 	"cercat/config"
 	"context"
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/gobwas/ws"
 	"github.com/gobwas/ws/wsutil"
-	log "github.com/sirupsen/logrus"
 )
 
 // the websocket stream from calidog
@@ -16,24 +17,22 @@ const certInput = "wss://certstream.calidog.io"
 // StartLoopCertStream gathers messages from CertStream
 func StartLoopCertStream(cfg *config.Configuration) {
 	dial := ws.Dialer{
-		ReadBufferSize:  8192,
-		WriteBufferSize: 512,
-		Timeout:         1 * time.Second,
+		ReadBufferSize:  2048,
+		WriteBufferSize: 128,
+		Timeout:         5 * time.Second,
 	}
 	for {
-		// conn, _, _, err := ws.DefaultDialer.Dial(context.Background(), certInput)
 		conn, _, _, err := dial.Dial(context.Background(), certInput)
 		if err != nil {
-			log.Warn(err)
-			log.Warn("Error connecting to CertStream! Sleeping a few seconds and reconnecting...")
+			cfg.Log.Warn("Error connecting to CertStream! Sleeping a few seconds and reconnecting")
 			time.Sleep(1 * time.Second)
-			// conn.Close()
 			continue
 		}
 		for {
 			msg, _, err := wsutil.ReadServerData(conn)
+			// fmt.Println(string(msg))
 			if err != nil {
-				log.Warn("Error reading message from CertStream")
+				cfg.Log.Warn(fmt.Sprintf("Error reading message from CertStream (%v)", strings.TrimSuffix(err.Error(), " ")))
 				break
 			}
 			cfg.Messages <- msg