From c412289758c535a7d9c506ca0a2c42d7314f077c Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Tue, 15 Jul 2025 17:21:46 -0400 Subject: [PATCH 1/2] feat: Support connection close/reset fault injection This change supports new `connection_close_count` and `connection_reset_count` fault injection via the `fault-settings` header, which enables connection error testing such as the `x-speakeasy-retries` extension `retryConnectionErrors` configuration. Enabling these injection will immediately close the connection before writing any response. The reset handling sets SO_LINGER to 0, which on most platforms should cause a TCP RST packet. --- .../middleware/connection_error_injector.go | 42 +++++++++++++++++++ internal/middleware/fault.go | 37 ++++++++++++++-- 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 internal/middleware/connection_error_injector.go diff --git a/internal/middleware/connection_error_injector.go b/internal/middleware/connection_error_injector.go new file mode 100644 index 0000000..55ffbae --- /dev/null +++ b/internal/middleware/connection_error_injector.go @@ -0,0 +1,42 @@ +package middleware + +import ( + "net" + "net/http" + + "github.com/lingrino/go-fault" +) + +var _ fault.Injector = (*ConnectionErrorInjector)(nil) + +// Injects a connection error by closing the connection immediately. +// This simulates a connection reset or close error. +type ConnectionErrorInjector struct { + // Enable to set SO_LINGER to 0 before closing, which will cause a TCP RST + // packet on most platforms when closing. + Reset bool +} + +func (i *ConnectionErrorInjector) Handler(_ http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hijacker, ok := w.(http.Hijacker) + + if !ok { + http.Error(w, "connection hijacking not supported", http.StatusInternalServerError) + return + } + + conn, _, err := hijacker.Hijack() + + if err != nil { + http.Error(w, "failed to hijack connection", http.StatusInternalServerError) + return + } + + if tcpConn, ok := conn.(*net.TCPConn); ok && i.Reset { + _ = tcpConn.SetLinger(0) // Best effort RST on close + } + + conn.Close() + }) +} diff --git a/internal/middleware/fault.go b/internal/middleware/fault.go index 3a0bfa9..8a4fc1d 100644 --- a/internal/middleware/fault.go +++ b/internal/middleware/fault.go @@ -17,12 +17,24 @@ type FaultSession struct { Settings FaultSettings } +// Describes the fault injection settings for a session. The fault chain is +// a series of fault injectors that are applied to the request in order. The +// order of faults is: +// - Delay +// - ConnectionClose +// - ConnectionReset +// - Reject +// - Error type FaultSettings struct { - // NOTE: The way these fields are ordered represents their precedence in the - // fault chain. + // Number of times to close the connection. + ConnectionCloseCount int `json:"connection_close_count"` + + // ConnectionResetCount is the number of times to reset the connection. + ConnectionResetCount int `json:"connection_reset_count"` // DelayMS is the number of milliseconds to delay the request. DelayMS int64 `json:"delay_ms"` + // DelayCount is the number of times to delay the request. DelayCount int `json:"delay_count"` @@ -36,6 +48,7 @@ type FaultSettings struct { // NOTE: Error injection only takes effect after all rejections have // resolved if both of these injectors are enabled. ErrorCount int `json:"error_count"` + // ErrorCode is the status code to return when the error injector is enabled. ErrorCode int `json:"error_code"` } @@ -81,6 +94,10 @@ func Fault(h http.Handler) http.Handler { var faults []fault.Injector + // Since multiple injectors can be enabled, need to count the number of + // requests based on prior injector counts + countOffset := 0 + if settings.DelayMS > 0 && reqCount < settings.DelayCount { inj, err := fault.NewSlowInjector(time.Millisecond * time.Duration(settings.DelayMS)) if err != nil { @@ -91,7 +108,20 @@ func Fault(h http.Handler) http.Handler { faults = append(faults, inj) } - countOffset := 0 + // Delay injector does not increase the count offset. + + if settings.ConnectionCloseCount > 0 && reqCount < settings.ConnectionCloseCount+countOffset { + faults = append(faults, &ConnectionErrorInjector{}) + } + + countOffset += settings.ConnectionCloseCount + + if settings.ConnectionResetCount > 0 && reqCount < settings.ConnectionResetCount+countOffset { + faults = append(faults, &ConnectionErrorInjector{Reset: true}) + } + + countOffset += settings.ConnectionResetCount + if settings.RejectCount > 0 && reqCount < settings.RejectCount+countOffset { inj, err := fault.NewRejectInjector() if err != nil { @@ -103,6 +133,7 @@ func Fault(h http.Handler) http.Handler { } countOffset += settings.RejectCount + if settings.ErrorCode > 0 && reqCount < (settings.ErrorCount+countOffset) { inj, err := fault.NewErrorInjector(settings.ErrorCode, fault.WithStatusText("Injected error")) if err != nil { From 6b41c43fea2546a03576761e2ae639c656d27dcf Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Tue, 5 Aug 2025 09:22:35 -0400 Subject: [PATCH 2/2] chore: Make fault injector just reset as reject already handles connection close injection --- internal/middleware/fault.go | 24 ++++++------------- ...on_error_injector.go => reset_injector.go} | 18 ++++++-------- 2 files changed, 14 insertions(+), 28 deletions(-) rename internal/middleware/{connection_error_injector.go => reset_injector.go} (60%) diff --git a/internal/middleware/fault.go b/internal/middleware/fault.go index 8a4fc1d..491d777 100644 --- a/internal/middleware/fault.go +++ b/internal/middleware/fault.go @@ -21,23 +21,19 @@ type FaultSession struct { // a series of fault injectors that are applied to the request in order. The // order of faults is: // - Delay -// - ConnectionClose -// - ConnectionReset +// - Reset // - Reject // - Error type FaultSettings struct { - // Number of times to close the connection. - ConnectionCloseCount int `json:"connection_close_count"` - - // ConnectionResetCount is the number of times to reset the connection. - ConnectionResetCount int `json:"connection_reset_count"` - // DelayMS is the number of milliseconds to delay the request. DelayMS int64 `json:"delay_ms"` // DelayCount is the number of times to delay the request. DelayCount int `json:"delay_count"` + // ResetCount is the number of times to reset the connection. + ResetCount int `json:"reset_count"` + // RejectCount is the number of times to reject the request without a response. // A value greater than 0 enables this fault injector. RejectCount int `json:"reject_count"` @@ -110,17 +106,11 @@ func Fault(h http.Handler) http.Handler { // Delay injector does not increase the count offset. - if settings.ConnectionCloseCount > 0 && reqCount < settings.ConnectionCloseCount+countOffset { - faults = append(faults, &ConnectionErrorInjector{}) - } - - countOffset += settings.ConnectionCloseCount - - if settings.ConnectionResetCount > 0 && reqCount < settings.ConnectionResetCount+countOffset { - faults = append(faults, &ConnectionErrorInjector{Reset: true}) + if settings.ResetCount > 0 && reqCount < settings.ResetCount+countOffset { + faults = append(faults, &ConnectionResetInjector{}) } - countOffset += settings.ConnectionResetCount + countOffset += settings.ResetCount if settings.RejectCount > 0 && reqCount < settings.RejectCount+countOffset { inj, err := fault.NewRejectInjector() diff --git a/internal/middleware/connection_error_injector.go b/internal/middleware/reset_injector.go similarity index 60% rename from internal/middleware/connection_error_injector.go rename to internal/middleware/reset_injector.go index 55ffbae..4bdca04 100644 --- a/internal/middleware/connection_error_injector.go +++ b/internal/middleware/reset_injector.go @@ -7,17 +7,13 @@ import ( "github.com/lingrino/go-fault" ) -var _ fault.Injector = (*ConnectionErrorInjector)(nil) - -// Injects a connection error by closing the connection immediately. -// This simulates a connection reset or close error. -type ConnectionErrorInjector struct { - // Enable to set SO_LINGER to 0 before closing, which will cause a TCP RST - // packet on most platforms when closing. - Reset bool -} +var _ fault.Injector = (*ConnectionResetInjector)(nil) + +// Injects a connection error by closing the connection immediately while also +// simulating a TCP RST via setting SO_LINGER to 0. +type ConnectionResetInjector struct{} -func (i *ConnectionErrorInjector) Handler(_ http.Handler) http.Handler { +func (i *ConnectionResetInjector) Handler(_ http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { hijacker, ok := w.(http.Hijacker) @@ -33,7 +29,7 @@ func (i *ConnectionErrorInjector) Handler(_ http.Handler) http.Handler { return } - if tcpConn, ok := conn.(*net.TCPConn); ok && i.Reset { + if tcpConn, ok := conn.(*net.TCPConn); ok { _ = tcpConn.SetLinger(0) // Best effort RST on close }