Skip to content

Commit 495af3a

Browse files
torcolvintorcolvin
committed
Merge remote-tracking branch 'origin/main' into ci-improve-shell-scripts
2 parents 14bac70 + 8a7f39c commit 495af3a

File tree

78 files changed

+1780
-1340
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+1780
-1340
lines changed

‎Jenkinsfile

Lines changed: 3 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ pipeline {
1111
EE_BUILD_TAG = "cb_sg_enterprise"
1212
SGW_REPO = "github.com/couchbase/sync_gateway"
1313
GH_ACCESS_TOKEN_CREDENTIAL = "github_cb-robot-sg_access_token"
14-
GO111MODULE = "on"
15-
GOCACHE = "${WORKSPACE}/.gocache"
1614
}
1715

1816
tools {
@@ -42,7 +40,7 @@ pipeline {
4240
sh "which go"
4341
sh "go version"
4442
sh "go env"
45-
sshagent(credentials: ['CB SG Robot Github SSH Key']) {
43+
sshagent(credentials: ['CB_SG_Robot_Github_SSH_Key']) {
4644
sh '''
4745
[ -d ~/.ssh ] || mkdir ~/.ssh && chmod 0700 ~/.ssh
4846
ssh-keyscan -t rsa,dsa github.com >> ~/.ssh/known_hosts
@@ -66,12 +64,6 @@ pipeline {
6664

6765
stage('Builds') {
6866
parallel {
69-
stage('Test compile') {
70-
steps {
71-
// run no tests but force them to be compiled
72-
sh "go test -run=- -count=1 ./..."
73-
}
74-
}
7567
stage('CE Linux') {
7668
steps {
7769
sh "GOOS=linux go build -o sync_gateway_ce-linux -v ${SGW_REPO}"
@@ -82,38 +74,6 @@ pipeline {
8274
sh "GOOS=linux go build -o sync_gateway_ee-linux -tags ${EE_BUILD_TAG} -v ${SGW_REPO}"
8375
}
8476
}
85-
stage('CE macOS') {
86-
// TODO: Remove skip
87-
when { expression { return false } }
88-
steps {
89-
withEnv(["PATH+GO=${GOPATH}/bin"]) {
90-
echo 'TODO: figure out why build issues are caused by gosigar'
91-
sh "GOOS=darwin go build -o sync_gateway_ce-darwin -v ${SGW_REPO}"
92-
}
93-
}
94-
}
95-
stage('EE macOS') {
96-
// TODO: Remove skip
97-
when { expression { return false } }
98-
steps {
99-
withEnv(["PATH+GO=${GOPATH}/bin"]) {
100-
echo 'TODO: figure out why build issues are caused by gosigar'
101-
sh "GOOS=darwin go build -o sync_gateway_ee-darwin -tags ${EE_BUILD_TAG} -v ${SGW_REPO}"
102-
}
103-
}
104-
}
105-
/* can't build windows with cgo
106-
stage('CE Windows') {
107-
steps {
108-
sh "GOOS=windows go build -o sync_gateway_ce-windows -v ${SGW_REPO}"
109-
}
110-
}
111-
stage('EE Windows') {
112-
steps {
113-
sh "GOOS=windows go build -o sync_gateway_ee-windows -tags ${EE_BUILD_TAG} -v ${SGW_REPO}"
114-
}
115-
}
116-
*/
11777
stage('Windows Service') {
11878
steps {
11979
sh "GOOS=windows go build -o sync_gateway_ce-windows-service -v ${SGW_REPO}/service/sg-windows/sg-service"
@@ -122,68 +82,6 @@ pipeline {
12282
}
12383
}
12484

125-
stage('Checks') {
126-
parallel {
127-
stage('gofmt') {
128-
steps {
129-
script {
130-
try {
131-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-gofmt', description: 'Running', status: 'PENDING')
132-
sh "which gofmt" // check if gofmt is installed
133-
sh "gofmt -d -e . | tee gofmt.out"
134-
sh "test -z \"\$(cat gofmt.out)\""
135-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-gofmt', description: 'OK', status: 'SUCCESS')
136-
} catch (Exception e) {
137-
sh "wc -l < gofmt.out | awk '{printf \$1}' > gofmt.count"
138-
script {
139-
env.GOFMT_COUNT = readFile 'gofmt.count'
140-
}
141-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-gofmt', description: "found "+env.GOFMT_COUNT+" problems", status: 'FAILURE')
142-
unstable("gofmt failed")
143-
}
144-
}
145-
}
146-
}
147-
stage('go vet') {
148-
steps {
149-
warnError(message: "go vet failed") {
150-
sh "go vet -tags ${EE_BUILD_TAG} ./..."
151-
}
152-
}
153-
}
154-
stage('go fix') {
155-
steps {
156-
warnError(message: "go fix failed") {
157-
sh "go tool fix -diff . | tee gofix.out"
158-
sh "test -z \"\$(cat gofix.out)\""
159-
}
160-
}
161-
}
162-
stage('errcheck') {
163-
steps {
164-
script {
165-
try {
166-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-errcheck', description: 'Running', status: 'PENDING')
167-
withEnv(["PATH+GO=${env.GOTOOLS}/bin"]) {
168-
sh "which errcheck" // check if errcheck is installed
169-
sh "errcheck ./... | tee errcheck.out"
170-
}
171-
sh "test -z \"\$(cat errcheck.out)\""
172-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-errcheck', description: 'OK', status: 'SUCCESS')
173-
} catch (Exception e) {
174-
sh "wc -l < errcheck.out | awk '{printf \$1}' > errcheck.count"
175-
script {
176-
env.ERRCHECK_COUNT = readFile 'errcheck.count'
177-
}
178-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-errcheck', description: "found "+env.ERRCHECK_COUNT+" unhandled errors", status: 'FAILURE')
179-
unstable("errcheck failed")
180-
}
181-
}
182-
}
183-
}
184-
}
185-
}
186-
18785
stage('Tests') {
18886
parallel {
18987
stage('Unit') {
@@ -285,41 +183,6 @@ pipeline {
285183
}
286184
}
287185
}
288-
289-
stage('LiteCore') {
290-
stages {
291-
stage('against CE') {
292-
// TODO: Remove skip
293-
when { expression { return false } }
294-
steps {
295-
echo 'Example of where we could run an alternate version of lite-core unit tests, or against a running SG CE'
296-
}
297-
}
298-
stage('against EE') {
299-
// CBG-2237 skipping stage due to regular litecore test segfaults
300-
when { expression { return false } }
301-
steps {
302-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-litecore-ee', description: 'Running LiteCore Tests', status: 'PENDING')
303-
sh 'touch verbose_litecore.out'
304-
sh 'touch verbose_litecore-sg_trace.out'
305-
script {
306-
withCredentials([sshUserPrivateKey(credentialsId: 'CB SG Robot Github SSH Key', keyFileVariable: 'KEY')]) {
307-
try {
308-
sh 'docker run --rm -v $KEY:/root/.ssh/id_rsa -v `pwd`/sync_gateway_ee-linux:/sync_gateway -v `pwd`/verbose_litecore.out:/output.out -v `pwd`/verbose_litecore-sg_trace.out:/tmp/sglog/sg_trace.log couchbase/sg-test-litecore:latest -legacy-config'
309-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-litecore-ee', description: 'EE with LiteCore Test Passed', status: 'SUCCESS')
310-
} catch (Exception e) {
311-
githubNotify(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", context: 'sgw-pipeline-litecore-ee', description: 'EE with LiteCore Test Failed', status: 'FAILURE')
312-
// archive verbose test logs in the event of a test failure
313-
archiveArtifacts artifacts: 'verbose_litecore*.out', fingerprint: false
314-
unstable("EE LIteCore Test Failed")
315-
}
316-
}
317-
}
318-
}
319-
}
320-
}
321-
}
322-
323186
stage('Integration') {
324187
stages {
325188
stage('main') {
@@ -328,23 +191,7 @@ pipeline {
328191
echo 'Queueing Integration test for branch "main" ...'
329192
// Queues up an async integration test run using default build params (main branch),
330193
// but waits up to an hour for batches of PR merges before actually running (via quietPeriod)
331-
build job: 'MasterIntegration', quietPeriod: 3600, wait: false
332-
}
333-
}
334-
335-
stage('PR') {
336-
// TODO: Remove skip
337-
when { expression { return false } }
338-
steps {
339-
// TODO: Read labels on PR for 'integration-test'
340-
// if present, run stage as separate GH status
341-
echo 'Example of where we can run integration tests for this commit'
342-
gitStatusWrapper(credentialsId: "${GH_ACCESS_TOKEN_CREDENTIAL}", description: 'Running EE Integration Test', failureDescription: 'EE Integration Test Failed', gitHubContext: 'sgw-pipeline-integration-ee', successDescription: 'EE Integration Test Passed') {
343-
echo "Waiting for integration test to finish..."
344-
// TODO: add commit parameter
345-
// Block the pipeline, but don't propagate a failure up to the top-level job - rely on gitStatusWrapper letting us know it failed
346-
build job: 'sync-gateway-integration-master', wait: true, propagate: false
347-
}
194+
build job: 'MainIntegration', quietPeriod: 3600, wait: false
348195
}
349196
}
350197
}
@@ -397,7 +244,6 @@ pipeline {
397244
}
398245
cleanup {
399246
cleanWs(disableDeferredWipeout: true)
400-
sh "go clean -cache"
401-
}
247+
}
402248
}
403249
}

‎base/bucket.go

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ package base
1010

1111
import (
1212
"context"
13+
"crypto/md5"
1314
"crypto/tls"
1415
"crypto/x509"
16+
"encoding/hex"
1517
"errors"
1618
"fmt"
1719
"io"
@@ -372,14 +374,25 @@ func getMaxTTL(ctx context.Context, store CouchbaseBucketStore) (int, error) {
372374
return bucketResponseWithMaxTTL.MaxTTLSeconds, nil
373375
}
374376

375-
// Get the Server UUID of the bucket, this is also known as the Cluster UUID
376-
func GetServerUUID(ctx context.Context, store CouchbaseBucketStore) (uuid string, err error) {
377-
respBytes, _, err := store.MgmtRequest(ctx, http.MethodGet, "/pools", "application/json", nil)
377+
// GetServerUUID returns Couchbase Server Cluster UUID on a timeout. If running against rosmar, do return an empty string.
378+
func GetServerUUID(ctx context.Context, bucket Bucket) (string, error) {
379+
gocbV2Bucket, err := AsGocbV2Bucket(bucket)
378380
if err != nil {
379-
return "", err
381+
return "", nil
382+
}
383+
// start a retry loop to get server ID
384+
worker := func() (bool, error, string) {
385+
respBytes, _, err := gocbV2Bucket.MgmtRequest(ctx, http.MethodGet, "/pools", "application/json", nil)
386+
if err != nil {
387+
return true, err, ""
388+
}
389+
390+
uuid, err := ParseClusterUUID(respBytes)
391+
return false, err, uuid
380392
}
381393

382-
return ParseClusterUUID(respBytes)
394+
err, uuid := RetryLoop(ctx, "Getting ServerUUID", worker, GetNewDatabaseSleeperFunc())
395+
return uuid, err
383396
}
384397

385398
func ParseClusterUUID(respBytes []byte) (string, error) {
@@ -522,3 +535,43 @@ func RequireNoBucketTTL(ctx context.Context, b Bucket) error {
522535

523536
return nil
524537
}
538+
539+
// GetSourceID returns the source ID for a bucket.
540+
func GetSourceID(ctx context.Context, bucket Bucket) (string, error) {
541+
// for rosmar bucket and testing, use the bucket name as the source ID to make it easier to identify the source
542+
gocbBucket, err := AsGocbV2Bucket(bucket)
543+
if err != nil {
544+
return bucket.GetName(), nil
545+
}
546+
547+
// If not overwriting the source ID, for rosmar, serverUUID would be ""
548+
serverUUID, err := GetServerUUID(ctx, gocbBucket)
549+
if err != nil {
550+
return "", err
551+
}
552+
bucketUUID, err := bucket.UUID()
553+
if err != nil {
554+
return "", err
555+
}
556+
return CreateEncodedSourceID(bucketUUID, serverUUID)
557+
}
558+
559+
// CreateEncodedSourceID will hash the bucket UUID and cluster UUID using md5 hash function then will base64 encode it
560+
// This function is in sync with xdcr implementation of UUIDstoDocumentSource https://github.com/couchbase/goxdcr/blob/dfba7a5b4251d93db46e2b0b4b55ea014218931b/hlv/hlv.go#L51
561+
func CreateEncodedSourceID(bucketUUID, clusterUUID string) (string, error) {
562+
md5Hash := md5.Sum([]byte(bucketUUID + clusterUUID))
563+
hexStr := hex.EncodeToString(md5Hash[:])
564+
source, err := HexToBase64(hexStr)
565+
if err != nil {
566+
return "", err
567+
}
568+
return string(source), nil
569+
}
570+
571+
// GetNewDatabaseSleeperFunc returns a sleeper function during database connection
572+
func GetNewDatabaseSleeperFunc() RetrySleeper {
573+
return CreateDoublingSleeperFunc(
574+
13, // MaxNumRetries approx 40 seconds total retry duration
575+
5, // InitialRetrySleepTimeMS
576+
)
577+
}

‎base/constants.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,15 @@ const (
157157
FromConnStrWarningThreshold = 10 * time.Second
158158
)
159159

160+
// SyncGatewayRawDocXattrs is a list of xattrs that Sync Gateway will fetch when reading a raw document.
161+
var SyncGatewayRawDocXattrs = []string{
162+
SyncXattrName,
163+
GlobalXattrName,
164+
VvXattrName,
165+
MouXattrName,
166+
VirtualDocumentXattr,
167+
}
168+
160169
const (
161170
DefaultScope = "_default"
162171
DefaultCollection = "_default"

‎base/error.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func (err *HTTPError) Error() string {
104104

105105
// HTTPErrorf creates an HTTPError with an http status code and a formatted message.
106106
func HTTPErrorf(status int, format string, args ...interface{}) *HTTPError {
107-
return NewHTTPError(status, fmt.Sprintf(format, args...))
107+
return NewHTTPError(status, fmt.Errorf(format, args...).Error())
108108
}
109109

110110
// NewHTTPError creates an HTTPError with an http status code and a message. Use HTTPErrorf for printf style messages.

‎base/logger_audit.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,12 @@ func (f AuditFields) merge(ctx context.Context, overwrites AuditFields) AuditFie
115115
return f
116116
}
117117

118+
// AuditEventIsEnabled checks if the given audit event ID is enabled for the current context.
119+
// This may be used to avoid the cost of some data processing/gathering ahead of an audit event.
120+
func AuditEventIsEnabled(ctx context.Context, id AuditID) bool {
121+
return auditLogger.Load().shouldLog(id, ctx)
122+
}
123+
118124
// Audit creates and logs an audit event for the given ID and a set of additional data associated with the request.
119125
func Audit(ctx context.Context, id AuditID, additionalData AuditFields) {
120126
var fields AuditFields

0 commit comments

Comments
 (0)