From 050f993cb60767e1d7ac5e605cbf0e7cda83c433 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 20 Nov 2023 14:41:52 +1100 Subject: [PATCH 1/4] Add signing parameters to cfn template --- .../conf/bin/bk-install-elastic-stack.sh | 39 +++++++++++++++++++ .../conf/bin/bk-install-elastic-stack.ps1 | 32 +++++++++++++++ templates/aws-stack.yml | 28 +++++++++++++ 3 files changed, 99 insertions(+) diff --git a/packer/linux/conf/bin/bk-install-elastic-stack.sh b/packer/linux/conf/bin/bk-install-elastic-stack.sh index 1199c4e5b..a499f77eb 100755 --- a/packer/linux/conf/bin/bk-install-elastic-stack.sh +++ b/packer/linux/conf/bin/bk-install-elastic-stack.sh @@ -254,6 +254,7 @@ else BUILDKITE_AGENT_TIMESTAMPS_LINES="false" BUILDKITE_AGENT_NO_ANSI_TIMESTAMPS="false" fi + echo Setting \$BUILDKITE_AGENT_NO_ANSI_TIMESTAMPS to \$BUILDKITE_AGENT_TIMESTAMP_LINES echo "BUILDKITE_AGENT_TIMESTAMP_LINES is $BUILDKITE_AGENT_TIMESTAMPS_LINES" echo "BUILDKITE_AGENT_NO_ANSI_TIMESTAMPS is $BUILDKITE_AGENT_NO_ANSI_TIMESTAMPS" @@ -292,6 +293,44 @@ tracing-backend=${BUILDKITE_AGENT_TRACING_BACKEND} cancel-grace-period=${BUILDKITE_AGENT_CANCEL_GRACE_PERIOD} EOF +if [[ -n "$BUILDKITE_AGENT_SIGNING_KEY_PATH" ]]; then + echo "Fetching signing key from ssm: $BUILDKITE_AGENT_SIGNING_KEY_PATH..." + + keyfile=/etc/buildkite-agent/signing-key.json + + aws ssm get-parameter \ + --name "$BUILDKITE_AGENT_SIGNING_KEY_PATH" \ + --with-decryption \ + --query Parameter.Value \ + --output text >"$keyfile" + + echo "Setting ownership of $keyfile to buildkite-agent..." + chown buildkite-agent: "$keyfile" + + echo "signing-jwks-file=$keyfile" >>/etc/buildkite-agent/buildkite-agent.cfg +fi + +if [[ -n "$BUILDKITE_AGENT_SIGNING_KEY_ID" ]]; then + echo "signing-jwks-key-id=$BUILDKITE_AGENT_SIGNING_KEY_ID" >>/etc/buildkite-agent/buildkite-agent.cfg +fi + +if [[ -n "$BUILDKITE_AGENT_VERIFICATION_KEY_PATH" ]]; then + echo "Fetching signing key from ssm: $BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." + + keyfile=/etc/buildkite-agent/verification-key.json + + aws ssm get-parameter \ + --name "$BUILDKITE_AGENT_VERIFICATION_KEY_PATH" \ + --with-decryption \ + --query Parameter.Value \ + --output text >"$keyfile" + + echo "Setting ownership of $keyfile to buildkite-agent..." + chown buildkite-agent: "$keyfile" + + echo "verification-jwks-file=$keyfile" >>/etc/buildkite-agent/buildkite-agent.cfg +fi + if [[ "${BUILDKITE_ENV_FILE_URL}" != "" ]]; then echo "Fetching env file from ${BUILDKITE_ENV_FILE_URL}..." /usr/local/bin/bk-fetch.sh "${BUILDKITE_ENV_FILE_URL}" /var/lib/buildkite-agent/env diff --git a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 index de2856c7d..634009d04 100644 --- a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 +++ b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 @@ -150,6 +150,38 @@ tracing-backend=${Env:BUILDKITE_AGENT_TRACING_BACKEND} "@ $OFS=" " +If (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_SIGNING_KEY_PATH)) { + Write-Output "Fetching signing key from ssm: $Env:BUILDKITE_AGENT_SIGNING_KEY_PATH..." + + $keyfile=C:\buildkite-agent\signing-key.json + + aws ssm get-parameter ` + --name "$Env:BUILDKITE_AGENT_SIGNING_KEY_PATH" ` + --with-decryption ` + --query Parameter.Value ` + --output text >"$keyfile" + + Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-file=$keyfile" +} + +if (![string]::IsNullOrEmpty)($Env:BUILDKITE_AGENT_SIGNING_KEY_ID) { + Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-key-id=$Env:BUILDKITE_AGENT_SIGNING_KEY_ID" +} + +if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH)) { + Write-Output "Fetching verification key from ssm: $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." + + $keyfile=C:\buildkite-agent\verification-key.json + + aws ssm get-parameter ` + --name "$Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH" ` + --with-decryption ` + --query Parameter.Value ` + --output text >"$keyfile" + + Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "verification-jwks-file=$keyfile" +} + nssm set lifecycled AppEnvironmentExtra +AWS_REGION=$Env:AWS_REGION nssm set lifecycled AppEnvironmentExtra +LIFECYCLED_HANDLER="C:\buildkite-agent\bin\stop-agent-gracefully.ps1" Restart-Service lifecycled diff --git a/templates/aws-stack.yml b/templates/aws-stack.yml index 8bca2c69d..7d8bed808 100644 --- a/templates/aws-stack.yml +++ b/templates/aws-stack.yml @@ -50,6 +50,9 @@ Metadata: - BuildkiteAgentScalerServerlessARN - BuildkiteAgentScalerVersion - LogRetentionDays + - BuildkiteAgentSigningKeySSMParameter + - BuildkiteAgentSigningKeyID + - BuildkiteAgentVerificationKeySSMParameter - Label: default: Network Configuration @@ -202,6 +205,25 @@ Parameters: - "opentelemetry" Default: "" + BuildkiteAgentSigningKeySSMParameter: + Description: Existing SSM Parameter Store path to the to a JSON Web Key Set (JWKS) containing a key to sign jobs with. + Type: String + Default: "" + AllowedPattern: "^$|^/[a-zA-Z0-9_.\\-/]+$" + ConstraintDescription: "Expects a leading forward slash" + + BuildkiteAgentSigningKeyID: + Description: The ID of the key in the JWKS to use for signing jobs. If not specified, and the JWKS contains only one key, that key will be used. + Type: String + Default: "" + + BuildkiteAgentVerificationKeySSMParameter: + Description: Existing SSM Parameter Store path to the to a JSON Web Key Set (JWKS) containing keys with which to verify jobs. + Type: String + Default: "" + AllowedPattern: "^$|^/[a-zA-Z0-9_.\\-/]+$" + ConstraintDescription: "Expects a leading forward slash" + BuildkiteAgentCancelGracePeriod: Description: The number of seconds a canceled or timed out job is given to gracefully terminate and upload its artifacts. Type: Number @@ -1218,6 +1240,9 @@ Resources: $Env:BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" $Env:BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" $Env:BUILDKITE_AGENT_TRACING_BACKEND="${BuildkiteAgentTracingBackend}" + $Env:BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" \ + $Env:BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" \ + $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" \ $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" $Env:BUILDKITE_QUEUE="${BuildkiteQueue}" $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS="${BuildkiteAgentEnableGitMirrors}" @@ -1276,6 +1301,9 @@ Resources: BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" \ BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" \ BUILDKITE_AGENT_TRACING_BACKEND="${BuildkiteAgentTracingBackend}" \ + BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" \ + BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" \ + BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" \ BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" \ BUILDKITE_AGENT_CANCEL_GRACE_PERIOD="${BuildkiteAgentCancelGracePeriod}" \ BUILDKITE_QUEUE="${BuildkiteQueue}" \ From aff273ba729e42940ca3c66a8b97560cbbe242ba Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 1 Dec 2023 13:07:56 +1100 Subject: [PATCH 2/4] Fix powershell syntax error --- packer/windows/conf/bin/bk-install-elastic-stack.ps1 | 2 +- templates/aws-stack.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 index 634009d04..3beed2ec0 100644 --- a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 +++ b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 @@ -164,7 +164,7 @@ If (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_SIGNING_KEY_PATH)) { Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-file=$keyfile" } -if (![string]::IsNullOrEmpty)($Env:BUILDKITE_AGENT_SIGNING_KEY_ID) { +if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_SIGNING_KEY_ID)) { Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-key-id=$Env:BUILDKITE_AGENT_SIGNING_KEY_ID" } diff --git a/templates/aws-stack.yml b/templates/aws-stack.yml index 7d8bed808..ca4979277 100644 --- a/templates/aws-stack.yml +++ b/templates/aws-stack.yml @@ -1240,9 +1240,9 @@ Resources: $Env:BUILDKITE_AGENT_TIMESTAMP_LINES="${BuildkiteAgentTimestampLines}" $Env:BUILDKITE_AGENT_EXPERIMENTS="${BuildkiteAgentExperiments}" $Env:BUILDKITE_AGENT_TRACING_BACKEND="${BuildkiteAgentTracingBackend}" - $Env:BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" \ - $Env:BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" \ - $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" \ + $Env:BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" + $Env:BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" + $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" $Env:BUILDKITE_QUEUE="${BuildkiteQueue}" $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS="${BuildkiteAgentEnableGitMirrors}" From 6dfef8499135d0f96c788bcb1b147832445e5375 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Wed, 6 Dec 2023 14:27:02 +1100 Subject: [PATCH 3/4] Add verification failure behaviour param --- packer/linux/conf/bin/bk-install-elastic-stack.sh | 4 ++++ packer/windows/conf/bin/bk-install-elastic-stack.ps1 | 4 ++++ templates/aws-stack.yml | 12 ++++++++++++ 3 files changed, 20 insertions(+) diff --git a/packer/linux/conf/bin/bk-install-elastic-stack.sh b/packer/linux/conf/bin/bk-install-elastic-stack.sh index a499f77eb..0af54428a 100755 --- a/packer/linux/conf/bin/bk-install-elastic-stack.sh +++ b/packer/linux/conf/bin/bk-install-elastic-stack.sh @@ -314,6 +314,10 @@ if [[ -n "$BUILDKITE_AGENT_SIGNING_KEY_ID" ]]; then echo "signing-jwks-key-id=$BUILDKITE_AGENT_SIGNING_KEY_ID" >>/etc/buildkite-agent/buildkite-agent.cfg fi +if [[ -n "$BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR" ]]; then + echo "verification-failure-behavior=$BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR" >>/etc/buildkite-agent/buildkite-agent.cfg +fi + if [[ -n "$BUILDKITE_AGENT_VERIFICATION_KEY_PATH" ]]; then echo "Fetching signing key from ssm: $BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." diff --git a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 index 3beed2ec0..828791308 100644 --- a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 +++ b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 @@ -168,6 +168,10 @@ if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_SIGNING_KEY_ID)) { Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-key-id=$Env:BUILDKITE_AGENT_SIGNING_KEY_ID" } +if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR)) { + Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "verification-failure-behavior=$Env:BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR" +} + if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH)) { Write-Output "Fetching verification key from ssm: $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." diff --git a/templates/aws-stack.yml b/templates/aws-stack.yml index ca4979277..a93efb17e 100644 --- a/templates/aws-stack.yml +++ b/templates/aws-stack.yml @@ -53,6 +53,7 @@ Metadata: - BuildkiteAgentSigningKeySSMParameter - BuildkiteAgentSigningKeyID - BuildkiteAgentVerificationKeySSMParameter + - BuildkiteAgentVerificationFailureBehavior - Label: default: Network Configuration @@ -224,6 +225,15 @@ Parameters: AllowedPattern: "^$|^/[a-zA-Z0-9_.\\-/]+$" ConstraintDescription: "Expects a leading forward slash" + BuildkiteAgentVerificationFailureBehavior: + Description: "How the agent should respond when a job signature fails verification" + Type: String + AllowedValues: + - "block" + - "warn" + - "" + Default: "" + BuildkiteAgentCancelGracePeriod: Description: The number of seconds a canceled or timed out job is given to gracefully terminate and upload its artifacts. Type: Number @@ -1243,6 +1253,7 @@ Resources: $Env:BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" $Env:BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" $Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" + $Env:BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR="${BuildkiteAgentVerificationFailureBehavior}" $Env:BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" $Env:BUILDKITE_QUEUE="${BuildkiteQueue}" $Env:BUILDKITE_AGENT_ENABLE_GIT_MIRRORS="${BuildkiteAgentEnableGitMirrors}" @@ -1304,6 +1315,7 @@ Resources: BUILDKITE_AGENT_SIGNING_KEY_PATH="${BuildkiteAgentSigningKeySSMParameter}" \ BUILDKITE_AGENT_SIGNING_KEY_ID="${BuildkiteAgentSigningKeyID}" \ BUILDKITE_AGENT_VERIFICATION_KEY_PATH="${BuildkiteAgentVerificationKeySSMParameter}" \ + BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR="${BuildkiteAgentVerificationFailureBehavior}" \ BUILDKITE_AGENT_RELEASE="${BuildkiteAgentRelease}" \ BUILDKITE_AGENT_CANCEL_GRACE_PERIOD="${BuildkiteAgentCancelGracePeriod}" \ BUILDKITE_QUEUE="${BuildkiteQueue}" \ From 949ade16b597aad836964baf4f902b3d7341c53f Mon Sep 17 00:00:00 2001 From: Pete Tomasik Date: Fri, 15 Aug 2025 15:37:15 -0400 Subject: [PATCH 4/4] Security fixes to scripts --- packer/linux/conf/bin/bk-install-elastic-stack.sh | 12 +++++++----- packer/windows/conf/bin/bk-install-elastic-stack.ps1 | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packer/linux/conf/bin/bk-install-elastic-stack.sh b/packer/linux/conf/bin/bk-install-elastic-stack.sh index 7ab6bca97..0b92d9c9d 100755 --- a/packer/linux/conf/bin/bk-install-elastic-stack.sh +++ b/packer/linux/conf/bin/bk-install-elastic-stack.sh @@ -310,8 +310,9 @@ if [[ -n "$BUILDKITE_AGENT_SIGNING_KEY_PATH" ]]; then --query Parameter.Value \ --output text >"$keyfile" - echo "Setting ownership of $keyfile to buildkite-agent..." - chown buildkite-agent: "$keyfile" + echo "Setting ownership and permissions for $keyfile..." + chown root:buildkite-agent "$keyfile" + chmod 640 "$keyfile" echo "signing-jwks-file=$keyfile" >>/etc/buildkite-agent/buildkite-agent.cfg fi @@ -325,7 +326,7 @@ if [[ -n "$BUILDKITE_AGENT_VERIFICATION_FAILURE_BEHAVIOR" ]]; then fi if [[ -n "$BUILDKITE_AGENT_VERIFICATION_KEY_PATH" ]]; then - echo "Fetching signing key from ssm: $BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." + echo "Fetching verification key from ssm: $BUILDKITE_AGENT_VERIFICATION_KEY_PATH..." keyfile=/etc/buildkite-agent/verification-key.json @@ -335,8 +336,9 @@ if [[ -n "$BUILDKITE_AGENT_VERIFICATION_KEY_PATH" ]]; then --query Parameter.Value \ --output text >"$keyfile" - echo "Setting ownership of $keyfile to buildkite-agent..." - chown buildkite-agent: "$keyfile" + echo "Setting ownership and permissions for $keyfile..." + chown root:buildkite-agent "$keyfile" + chmod 640 "$keyfile" echo "verification-jwks-file=$keyfile" >>/etc/buildkite-agent/buildkite-agent.cfg fi diff --git a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 index a9163e910..e5fcf8f8a 100644 --- a/packer/windows/conf/bin/bk-install-elastic-stack.ps1 +++ b/packer/windows/conf/bin/bk-install-elastic-stack.ps1 @@ -172,6 +172,10 @@ If (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_SIGNING_KEY_PATH)) { --query Parameter.Value ` --output text >"$keyfile" + Write-Output "Setting permissions for $keyfile..." + # Remove inheritance and set explicit permissions: Administrators=FullControl, buildkite-agent=Read + icacls "$keyfile" /inheritance:r /grant:r "Administrators:F" /grant:r "buildkite-agent:R" + Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "signing-jwks-file=$keyfile" } @@ -194,6 +198,9 @@ if (![string]::IsNullOrEmpty($Env:BUILDKITE_AGENT_VERIFICATION_KEY_PATH)) { --query Parameter.Value ` --output text >"$keyfile" + Write-Output "Setting permissions for $keyfile..." + # Remove inheritance and set explicit permissions: Administrators=FullControl, buildkite-agent=Read + icacls "$keyfile" /inheritance:r /grant:r "Administrators:F" /grant:r "buildkite-agent:R" Add-Content -Path C:\buildkite-agent\buildkite-agent.cfg -Value "verification-jwks-file=$keyfile" }