Skip to content

Commit e61e003

Browse files
IssifIssif
committed
update deps
Signed-off-by: Issif <issif_github@gadz.org>
1 parent a4de4af commit e61e003

File tree

14 files changed

+521
-162
lines changed

14 files changed

+521
-162
lines changed

‎.goreleaser.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ builds:
66
goos:
77
- linux
88
- darwin
9-
- windows
109
goarch:
1110
- amd64
1211
checksum:

‎README.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ Workers: 20 # Number of workers to compare the certificates with the Regexp
3838
TakeScreenshot: false # Try to take a screenshot of the website
3939
ScreenshotsFolder: "." # Folder to store the screenshots
4040
IgnoreOlderThan: 10 # Ignore domains older than
41+
WrongLinksThreshold: 20 # Ignore domains with less broken links than
4142
```
4243

4344
### With env vars
@@ -50,6 +51,7 @@ IgnoreOlderThan: 10 # Ignore domains older than
5051
- **TAKESCREENSHOT**: Try to take a screenshot of the website
5152
- **SCREENSHOTSFOLDER**: Folder to store the screenshots
5253
- **IGNOREOLDERTHAN**: Ignore domains older than
54+
- **WRONGLINKSTHRESHOLD**: Ignore domains with less broken links than
5355

5456
## Run
5557

@@ -72,8 +74,8 @@ docker run -d -e SLACKWEBHOOKURL=https://hooks.slack.com/services/XXXXX -e REGEX
7274
## Logs
7375

7476
```bash
75-
INFO[0005] A certificate for 'xxxx.fr' has been issued : {"domain":"xxxx.fr","SAN":["xxxx.fr","www.xxxx.fr"],"issuer":"Let's Encrypt","Addresses":["X.X.X.129"]}
76-
INFO[0008] A certificate for 'xxxx.fr' has been issued : {"domain":"xxxx.fr","SAN":["xxxx.fr","www.xxxx.fr"],"issuer":"Let's Encrypt","Addresses":["X.X.X.116"]}
77+
INFO[2023-09-13 15:18:17] A suspicious phishing website 'xxxx.com' has been detected
78+
INFO[2023-09-13 15:18:17] A confirmed phishing website 'xxxx.fr' has been detected
7779
```
7880

7981
## Profiles, Traces and Metrics

‎cmd/main.go

Lines changed: 15 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,13 @@ import (
88
"cercat/pkg/worker"
99
"fmt"
1010
"net/http"
11+
_ "net/http/pprof"
1112
"os"
1213
"path/filepath"
1314

14-
_ "net/http/pprof"
15-
15+
"github.com/alecthomas/kingpin"
1616
"github.com/arl/statsviz"
1717
"github.com/pkg/errors"
18-
"gopkg.in/alecthomas/kingpin.v2"
1918
)
2019

2120
func init() {
@@ -38,32 +37,25 @@ func main() {
3837
cfg := config.CreateConfig(configFile)
3938
// fmt.Printf("%#v\n", cfg)
4039
for i := 0; i < cfg.Workers; i++ {
41-
go worker.RunCertCheckWorker(cfg)
40+
go worker.RunCertCheckWorker()
4241
}
43-
go runNotifierWorker(cfg)
44-
certstream.StartLoopCertStream(cfg)
42+
go runNotifierWorker()
43+
certstream.StartLoopCertStream()
4544
}
4645

4746
// runNotifierWorker is a worker that receives cert, depduplicates and sends to Slack the event
48-
func runNotifierWorker(cfg *config.Configuration) {
47+
func runNotifierWorker() {
48+
cfg := config.GetConfig()
49+
4950
for {
5051
result := <-cfg.Buffer
51-
duplicate := false
52-
cfg.PreviousCerts.Do(func(d interface{}) {
53-
if result.Domain == d {
54-
duplicate = true
55-
}
56-
})
57-
if !duplicate {
58-
cfg.PreviousCerts = cfg.PreviousCerts.Prev()
59-
cfg.PreviousCerts.Value = result.Domain
60-
// j, _ := json.Marshal(result)
61-
cfg.Log.Infof("A certificate for '%v' has been issued by %v\n", result.Domain, result.Registrar)
62-
if cfg.SlackWebHookURL != "" {
63-
go func(c *config.Configuration, r *model.Result) {
64-
slack.NewPayload(c, result).Post(c)
65-
}(cfg, result)
66-
}
52+
53+
// j, _ := json.Marshal(result)
54+
cfg.Log.Infof("A %v phishing website '%v' has been detected\n", result.Status, result.Domain)
55+
if cfg.SlackWebHookURL != "" {
56+
go func(c *config.Configuration, r *model.Result) {
57+
slack.NewPayload(c, result).Post(c)
58+
}(cfg, result)
6759
}
6860
}
6961
}

‎config/config.go

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,50 @@ import (
1515

1616
// Configuration represents a configuration element
1717
type Configuration struct {
18-
Workers int
19-
SlackWebHookURL string
20-
SlackIconURL string
21-
SlackUsername string
22-
Regexp string
23-
ScreenshotsFolder string
24-
TakeScreenshot bool
25-
IgnoreOlderThan int
26-
RegIP string
27-
RegexpC *regexp.Regexp
28-
PreviousCerts *ring.Ring
29-
Messages chan []byte
30-
Buffer chan *model.Result
31-
Homoglyph map[string]string
32-
Log *logrus.Logger
18+
Workers int
19+
SlackWebHookURL string
20+
SlackIconURL string
21+
SlackUsername string
22+
Regexp string
23+
ScreenshotsFolder string
24+
TakeScreenshot bool
25+
IgnoreOlderThan int
26+
WrongLinksThreshold int
27+
RegIP string
28+
RegexpC *regexp.Regexp
29+
PreviousCerts *ring.Ring
30+
Messages chan []byte
31+
Buffer chan *model.Result
32+
Homoglyph map[string]string
33+
Log *logrus.Logger
34+
}
35+
36+
var config *Configuration
37+
38+
func init() {
39+
config = new(Configuration)
40+
}
41+
42+
func GetConfig() *Configuration {
43+
return config
3344
}
3445

3546
// CreateConfig provides a Configuration
3647
func CreateConfig(configFile *string) *Configuration {
37-
c := &Configuration{
48+
config = &Configuration{
3849
Workers: 50,
3950
Homoglyph: homoglyph.GetHomoglyphMap(),
40-
PreviousCerts: ring.New(20),
51+
PreviousCerts: ring.New(200),
4152
Messages: make(chan []byte, 100),
4253
Buffer: make(chan *model.Result, 100),
4354
Log: logrus.New(),
4455
}
4556

46-
c.Log.SetFormatter(&logrus.TextFormatter{
57+
config.Log.SetFormatter(&logrus.TextFormatter{
4758
FullTimestamp: true,
4859
TimestampFormat: "2006-01-02 15:04:05",
4960
})
50-
c.Log.SetOutput(os.Stdout)
61+
config.Log.SetOutput(os.Stdout)
5162

5263
v := viper.New()
5364
v.SetDefault("SlackWebhookURL", "")
@@ -58,6 +69,7 @@ func CreateConfig(configFile *string) *Configuration {
5869
v.SetDefault("TakeScreenshot", false)
5970
v.SetDefault("ScreenshotsFolder", ".")
6071
v.SetDefault("IgnoreOlderThan", 10)
72+
v.SetDefault("WrongLinksThreshold", 20)
6173

6274
if *configFile != "" {
6375
d, f := path.Split(*configFile)
@@ -68,24 +80,24 @@ func CreateConfig(configFile *string) *Configuration {
6880
v.AddConfigPath(d)
6981
err := v.ReadInConfig()
7082
if err != nil {
71-
c.Log.Fatalf("[ERROR] : Error when reading config file : %v\n", err)
83+
config.Log.Fatalf("[ERROR] : Error when reading config file : %v\n", err)
7284
}
7385
}
7486
v.AutomaticEnv()
75-
v.Unmarshal(c)
87+
v.Unmarshal(config)
7688

77-
if c.SlackUsername == "" {
78-
c.SlackUsername = "Cercat"
89+
if config.SlackUsername == "" {
90+
config.SlackUsername = "Cercat"
7991
}
8092

81-
if c.Regexp == "" {
82-
c.Log.Fatal("Regexp can't be empty")
93+
if config.Regexp == "" {
94+
config.Log.Fatal("Regexp can't be empty")
8395
}
8496

85-
reg, err := regexp.Compile(c.Regexp)
97+
reg, err := regexp.Compile(config.Regexp)
8698
if err != nil {
87-
c.Log.Fatal("Bad regexp")
99+
config.Log.Fatal("Bad regexp")
88100
}
89-
c.RegexpC = reg
90-
return c
101+
config.RegexpC = reg
102+
return config
91103
}

‎example.yaml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
---
2-
SlackWebhookURL: "" #Slack Webhook URL
3-
SlackIconURL: "" #Slack Icon (Avatar) URL
4-
SlackUsername: "" #Slack Username
5-
Regexp: ".*\\.fr$" #Regexp to match. Can't be empty. It uses Golang regexp format
2+
SlackWebhookURL: "" # Slack Webhook URL
3+
SlackIconURL: "" # Slack Icon (Avatar) URL
4+
SlackUsername: "Cercat" # Slack Username
5+
Regexp: ".*\\.fr$" # Regexp to match. Can't be empty. It uses Golang regexp format
6+
Workers: 20 # Number of workers to compare the certificates with the Regexp
7+
TakeScreenshot: false # Try to take a screenshot of the website
8+
ScreenshotsFolder: "." # Folder to store the screenshots
9+
IgnoreOlderThan: 10 # Ignore domains older than
10+
WrongLinksThreshold: 20 # Ignore domains with less broken links than

‎go.mod

Lines changed: 49 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,71 @@
11
module cercat
22

3-
go 1.18
3+
go 1.24
4+
5+
toolchain go1.24.0
46

57
require (
6-
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
7-
github.com/arl/statsviz v0.5.1
8-
github.com/chromedp/cdproto v0.0.0-20230126215531-b7d95b322d50 // indirect
9-
github.com/chromedp/chromedp v0.8.7
10-
github.com/gobwas/ws v1.1.0
11-
github.com/gorilla/websocket v1.5.0 // indirect
8+
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
9+
github.com/arl/statsviz v0.6.0
10+
github.com/chromedp/cdproto v0.0.0-20250307225615-b9fffb6d31ad // indirect
11+
github.com/chromedp/chromedp v0.13.1
12+
github.com/gobwas/ws v1.4.0
13+
github.com/gorilla/websocket v1.5.3 // indirect
1214
github.com/picatz/homoglyphr v0.0.0-20180114170158-6e9a0e190785
1315
github.com/pkg/errors v0.9.1
14-
github.com/sirupsen/logrus v1.9.0
15-
github.com/spf13/viper v1.15.0
16-
golang.org/x/net v0.5.0
17-
gopkg.in/alecthomas/kingpin.v2 v2.2.6
16+
github.com/sirupsen/logrus v1.9.3
17+
github.com/spf13/viper v1.19.0
18+
golang.org/x/net v0.37.0
1819
)
1920

2021
require (
22+
github.com/alecthomas/kingpin v2.2.6+incompatible
23+
github.com/gocolly/colly v1.2.0
2124
github.com/jpillora/go-tld v1.2.1
22-
github.com/likexian/whois v1.14.4
23-
github.com/likexian/whois-parser v1.24.2
25+
github.com/likexian/whois v1.15.6
26+
github.com/likexian/whois-parser v1.24.20
2427
)
2528

2629
require (
27-
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect
28-
github.com/chromedp/sysutil v1.0.0 // indirect
29-
github.com/fsnotify/fsnotify v1.6.0 // indirect
30+
github.com/PuerkitoBio/goquery v1.10.2 // indirect
31+
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
32+
github.com/andybalholm/cascadia v1.3.3 // indirect
33+
github.com/antchfx/htmlquery v1.3.4 // indirect
34+
github.com/antchfx/xmlquery v1.4.4 // indirect
35+
github.com/antchfx/xpath v1.3.3 // indirect
36+
github.com/chromedp/sysutil v1.1.0 // indirect
37+
github.com/fsnotify/fsnotify v1.8.0 // indirect
38+
github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874 // indirect
39+
github.com/gobwas/glob v0.2.3 // indirect
3040
github.com/gobwas/httphead v0.1.0 // indirect
3141
github.com/gobwas/pool v0.2.1 // indirect
42+
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
43+
github.com/golang/protobuf v1.5.4 // indirect
3244
github.com/hashicorp/hcl v1.0.0 // indirect
3345
github.com/josharian/intern v1.0.0 // indirect
34-
github.com/likexian/gokit v0.25.9 // indirect
35-
github.com/magiconair/properties v1.8.7 // indirect
36-
github.com/mailru/easyjson v0.7.7 // indirect
46+
github.com/kennygrant/sanitize v1.2.4 // indirect
47+
github.com/likexian/gokit v0.25.15 // indirect
48+
github.com/magiconair/properties v1.8.9 // indirect
49+
github.com/mailru/easyjson v0.9.0 // indirect
3750
github.com/mitchellh/mapstructure v1.5.0 // indirect
38-
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
39-
github.com/spf13/afero v1.9.3 // indirect
40-
github.com/spf13/cast v1.5.0 // indirect
51+
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
52+
github.com/sagikazarmark/locafero v0.7.0 // indirect
53+
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
54+
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
55+
github.com/sourcegraph/conc v0.3.0 // indirect
56+
github.com/spf13/afero v1.12.0 // indirect
57+
github.com/spf13/cast v1.7.1 // indirect
4158
github.com/spf13/jwalterweatherman v1.1.0 // indirect
42-
github.com/spf13/pflag v1.0.5 // indirect
43-
github.com/subosito/gotenv v1.4.2 // indirect
44-
golang.org/x/sys v0.4.0 // indirect
45-
golang.org/x/text v0.6.0 // indirect
59+
github.com/spf13/pflag v1.0.6 // indirect
60+
github.com/subosito/gotenv v1.6.0 // indirect
61+
github.com/temoto/robotstxt v1.1.2 // indirect
62+
go.uber.org/atomic v1.11.0 // indirect
63+
go.uber.org/multierr v1.11.0 // indirect
64+
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
65+
golang.org/x/sys v0.31.0 // indirect
66+
golang.org/x/text v0.23.0 // indirect
67+
google.golang.org/appengine v1.6.8 // indirect
68+
google.golang.org/protobuf v1.36.5 // indirect
4669
gopkg.in/ini.v1 v1.67.0 // indirect
4770
gopkg.in/yaml.v3 v3.0.1 // indirect
4871
)

0 commit comments

Comments
 (0)