Inconsistent result with same API call to Prometheus via client_golang #1836
Description
Hi there, we build a golang service and use client_golang to fetch data from a thanos backend. But recently we find when a request was repeatedly, some data points would be NULL sometimes but normal sometimes. Then we checked the backend by other client (grafana, web UI), the data series was good without any NULL, and we could not reproduce this issue except using client_golang.
Library version: v1.22.0.
Query code block sample:
endpoint: https://prometheus-tech.ingress.<<domain>>
func setPrometheusAPI(promClient *PrometheusClient, cred Credential) error {
customRT := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 10 * time.Millisecond,
}).DialContext,
DisableKeepAlives: true,
IdleConnTimeout: 90 * time.Millisecond,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 60,
MaxConnsPerHost: 60,
}
client, err := api.NewClient(api.Config{
Address: promClient.Endpoint,
RoundTripper: cfg.NewBasicAuthRoundTripper(cfg.NewInlineSecret(cred.UserName), cfg.NewInlineSecret(cred.Password), customRT),
})
if err != nil {
log.Error("Error while creating prometheus go client: ", err)
return err
}
promClient.API = v1.NewAPI(client)
return nil
}
func (promClient *PrometheusClient) QueryRange(query string, start time.Time, end time.Time, step time.Duration) (model.Value, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(config.Params.Value("prometheusQueryTimeout"))*time.Second)
defer cancel()
r := v1.Range{
Start: start,
End: end,
Step: step,
}
result, warnings, err := promClient.API.QueryRange(ctx, query, r)
if err != nil {
log.Error(err)
log.Debug("[PromQL] cluster: ", promClient.ClusterName, ", query: ", query, ", step: ", step,
", start: ", start.Format(time.RFC3339), ", end: ", end.Format(time.RFC3339))
// refresh prom client and retry if error is 401
if strings.Contains(err.Error(), ClientErrMsg) {
err = refreshPromClientInCache(promClient, promClient.ClusterName)
if err != nil {
log.Error("Error while refreshing prom client in cache: ", err)
return nil, err
}
result, warnings, err = promClient.API.QueryRange(ctx, query, r)
if err != nil {
log.Error("Error while performing QueryRange: ", err)
return nil, err
}
} else {
return nil, err
}
}
if len(warnings) > 0 {
log.Warn("Warnings while performing QueryRange: ", warnings)
}
return result, nil
}
And a typical result request for same promQL shows like
start: 2025-05-01T00:00:00Z, end: 2025-05-16T06:00:00Z, step: 1h0m0s,"time":"2025-05-19T07:21:00Z"
0.0021250095625 @[1747022400]\n0.00212827828125 @[1747026000]\n0.0021315708906249997 @[1747029600]\n0.002134853359375 @[1747033200]\n0.002138032359375 @[1747036800]\n0.002142065921875 @[1747040400]\n0.0021456414375 @[1747044000]\n0.00215883559375 @[1747051200]\n0.0021628825625 @[1747054800]\n0.002166302703125 @[1747058400]\n0.002169638625 @[1747062000]\n0.0021735838437499998 @[1747065600]\n0.0021779193906249997 @[1747069200]\n0.0022344377343749998 @[1747072800]\n0.002252548890625 @[1747076400]\n0.0022559645624999998 @[1747080000]\n0.002259400859375 @[1747083600]\n0.0022628797812499998 @[1747087200]\n0.0022662251562499997 @[1747090800]\n0.002269592875 @[1747094400]\n0.002171072234375 @[1747098000]\n0.0021403679687499997 @[1747101600]\n0.002143655078125 @[1747105200]\n0.00214693359375 @[1747108800]\n0.0021501816874999998 @[1747112400]\n0.0021534907968749997 @[1747116000]\n0.0021567476562499997 @[1747119600]\n0.002164359484375 @[1747126800]\n0.0021676814843749996 @[1747130400]\n0.0021748345781249998 @[1747134000]\n0.0021818470781249997 @[1747137600]\n0.002185245390625 @[1747141200]\n0.0021886263437499998 @[1747144800]\n0.0021919591718749997 @[1747148400]\n0.0021967489843749996 @[1747152000]\n0.002200361109375 @[1747155600]\n0.00225745265625 @[1747159200]\n0.0022758254062499998 @[1747162800]\n0.002279258953125 @[1747166400]\n0.002282716390625 @[1747170000]\n0.00228619084375 @[1747173600]\n0.00228964621875 @[1747177200]\n0.002293150578125 @[1747180800]\n0.0022966420468749998 @[1747184400]\n0.002303738078125 @[1747191600]\n0.00230729365625 @[1747195200]\n0.00231087621875 @[1747198800]\n0.002314403265625 @[1747202400]\n0.002317956953125 @[1747206000]\n0.00232153178125 @[1747209600]\n0.00232509921875 @[1747213200]\n0.0023286640781249998 @[1747216800]\n0.002217384640625 @[1747220400]\n0.002192085671875 @[1747224000]\n0.0021954635312499996 @[1747227600]\n0.002198889515625 @[1747231200]\n0.002202331484375 @[1747234800]\n0.00220610328125 @[1747238400]\n0.002211789421875 @[1747242000]\n0.002210605546875 @[1747249200]\n0.0021444570468749998 @[1747252800]\n0.0021476455 @[1747256400]\n0.0021509283125 @[1747260000]\n0.0021541012968749997 @[1747263600]\n0.002157292328125 @[1747267200]\n0.002160481296875 @[1747270800]\n0.00216412796875 @[1747274400]\n0.002168208796875 @[1747278000]\n0.0021715223749999998 @[1747281600]\n0.0021748399062499995 @[1747285200]\n0.0021780941875 @[1747288800]\n0.0021813238906249997 @[1747292400]\n0.002184592265625 @[1747296000]\n0.00218799109375 @[1747299600]\n0.002192305328125 @[1747303200]\n0.002195634890625 @[1747306800]\n0.002199084421875 @[1747310400]\n0.002202419140625 @[1747314000]\n0.00220575128125 @[1747317600]\n0.002212976390625 @[1747321200]\n0.00221734975 @[1747324800]\n0.002220684125 @[1747328400]\n0.00217589728125 @[1747332000]\n0.002156833078125 @[1747335600]
start: 2025-05-01T00:00:00Z, end: 2025-05-16T06:00:00Z, step: 1h0m0s,"time":"2025-05-19T07:21:01Z"
0.0021250095625 @[1747022400]\n0.00212827828125 @[1747026000]\n0.0021315708906249997 @[1747029600]\n0.002134853359375 @[1747033200]\n0.002138032359375 @[1747036800]\n0.002142065921875 @[1747040400]\n0.0021456414375 @[1747044000]\n0.0021456414375 @[1747047600]\n0.00215883559375 @[1747051200]\n0.0021628825625 @[1747054800]\n0.002166302703125 @[1747058400]\n0.002169638625 @[1747062000]\n0.0021735838437499998 @[1747065600]\n0.0021779193906249997 @[1747069200]\n0.0022344377343749998 @[1747072800]\n0.002252548890625 @[1747076400]\n0.0022559645624999998 @[1747080000]\n0.002259400859375 @[1747083600]\n0.0022628797812499998 @[1747087200]\n0.0022662251562499997 @[1747090800]\n0.002269592875 @[1747094400]\n0.002171072234375 @[1747098000]\n0.0021403679687499997 @[1747101600]\n0.002143655078125 @[1747105200]\n0.00214693359375 @[1747108800]\n0.0021501816874999998 @[1747112400]\n0.0021534907968749997 @[1747116000]\n0.0021567476562499997 @[1747119600]\n0.0021567476562499997 @[1747123200]\n0.002164359484375 @[1747126800]\n0.0021676814843749996 @[1747130400]\n0.0021748345781249998 @[1747134000]\n0.0021818470781249997 @[1747137600]\n0.002185245390625 @[1747141200]\n0.0021886263437499998 @[1747144800]\n0.0021919591718749997 @[1747148400]\n0.0021967489843749996 @[1747152000]\n0.002200361109375 @[1747155600]\n0.00225745265625 @[1747159200]\n0.0022758254062499998 @[1747162800]\n0.002279258953125 @[1747166400]\n0.002282716390625 @[1747170000]\n0.00228619084375 @[1747173600]\n0.00228964621875 @[1747177200]\n0.002293150578125 @[1747180800]\n0.0022966420468749998 @[1747184400]\n0.002303738078125 @[1747191600]\n0.00230729365625 @[1747195200]\n0.00231087621875 @[1747198800]\n0.002314403265625 @[1747202400]\n0.002317956953125 @[1747206000]\n0.00232153178125 @[1747209600]\n0.00232509921875 @[1747213200]\n0.0023286640781249998 @[1747216800]\n0.002217384640625 @[1747220400]\n0.002192085671875 @[1747224000]\n0.0021954635312499996 @[1747227600]\n0.002198889515625 @[1747231200]\n0.002202331484375 @[1747234800]\n0.00220610328125 @[1747238400]\n0.002211789421875 @[1747242000]\n0.002210605546875 @[1747249200]\n0.0021444570468749998 @[1747252800]\n0.0021476455 @[1747256400]\n0.0021509283125 @[1747260000]\n0.0021541012968749997 @[1747263600]\n0.002157292328125 @[1747267200]\n0.002160481296875 @[1747270800]\n0.00216412796875 @[1747274400]\n0.002168208796875 @[1747278000]\n0.0021715223749999998 @[1747281600]\n0.0021748399062499995 @[1747285200]\n0.0021780941875 @[1747288800]\n0.0021813238906249997 @[1747292400]\n0.002184592265625 @[1747296000]\n0.00218799109375 @[1747299600]\n0.002192305328125 @[1747303200]\n0.002195634890625 @[1747306800]\n0.002199084421875 @[1747310400]\n0.002202419140625 @[1747314000]\n0.00220575128125 @[1747317600]\n0.002212976390625 @[1747321200]\n0.00221734975 @[1747324800]\n0.002220684125 @[1747328400]\n0.00217589728125 @[1747332000]\n0.002156833078125 @[1747335600]
This result is directly printed of retrieved data. And you could find clearly that the first data series is lack of some points, e.g. at time stamp @[1747047600], @[1747123200]...
If there is any details insufficient, we could provide more information, THX!