Skip to content

Commit 1506c39

Browse files
brendankellyibmvburckhardt
brendankellyibm
and
vburckhardt
authored
feat: discovery scripts to check for existing project before creating (#47)
* Updated discovery scripts to be more robust Signed-off-by: brendankellyibm <brendank@ie.ibm.com> * fix retry flag * fix more retry flags * feat: optionally run ci pipeline * feat: optionally run ci pipeline --------- Signed-off-by: brendankellyibm <brendank@ie.ibm.com> Co-authored-by: Vincent Burckhardt <vincent.burckhardt@ie.ibm.com>
1 parent a6a8653 commit 1506c39

12 files changed

+130
-68
lines changed

‎common-dev-assets

‎solutions/banking/main.tf

Lines changed: 17 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,6 @@ resource "ibm_resource_instance" "cd_instance" {
7575
resource_group_id = data.ibm_resource_group.toolchain_resource_group_id.id
7676
}
7777

78-
# create watsonX.AI user - do we need this?
79-
# module "configure_user" {
80-
# source = "./configure_user"
81-
# resource_group_id = module.resource_group.resource_group_id
82-
# }
83-
8478
# create watsonx.AI project
8579
module "configure_project" {
8680
watsonx_admin_api_key = var.watsonx_admin_api_key != null ? var.watsonx_admin_api_key : var.ibmcloud_api_key
@@ -106,9 +100,12 @@ resource "null_resource" "discovery_project_creation" {
106100
}
107101

108102
provisioner "local-exec" {
109-
command = "${path.module}/watson-scripts/discovery-project-creation.sh \"${local.sensitive_tokendata}\" \"${local.watsonx_discovery_url}\""
103+
command = "${path.module}/watson-scripts/discovery-project-creation.sh \"${local.watsonx_discovery_url}\""
110104
interpreter = ["/bin/bash", "-c"]
111105
quiet = true
106+
environment = {
107+
IAM_TOKEN = local.sensitive_tokendata
108+
}
112109
}
113110
}
114111

@@ -120,9 +117,12 @@ resource "null_resource" "discovery_collection_creation" {
120117
}
121118

122119
provisioner "local-exec" {
123-
command = "${path.module}/watson-scripts/discovery-collection-creation.sh \"${local.sensitive_tokendata}\" \"${local.watsonx_discovery_url}\""
120+
command = "${path.module}/watson-scripts/discovery-collection-creation.sh \"${local.watsonx_discovery_url}\""
124121
interpreter = ["/bin/bash", "-c"]
125122
quiet = true
123+
environment = {
124+
IAM_TOKEN = local.sensitive_tokendata
125+
}
126126
}
127127
}
128128

@@ -134,9 +134,12 @@ resource "null_resource" "discovery_file_upload" {
134134
}
135135

136136
provisioner "local-exec" {
137-
command = "${path.module}/watson-scripts/discovery-file-upload.sh \"${local.sensitive_tokendata}\" \"${local.watsonx_discovery_url}\" \"${path.module}/artifacts/WatsonDiscovery\" "
137+
command = "${path.module}/watson-scripts/discovery-file-upload.sh \"${local.watsonx_discovery_url}\" \"${path.module}/artifacts/WatsonDiscovery\" "
138138
interpreter = ["/bin/bash", "-c"]
139139
quiet = true
140+
environment = {
141+
IAM_TOKEN = local.sensitive_tokendata
142+
}
140143
}
141144
}
142145

@@ -147,39 +150,15 @@ resource "null_resource" "assistant_project_creation" {
147150
}
148151

149152
provisioner "local-exec" {
150-
command = "${path.module}/watson-scripts/assistant-project-creation.sh \"${local.sensitive_tokendata}\" \"${local.watsonx_assistant_url}\""
153+
command = "${path.module}/watson-scripts/assistant-project-creation.sh \"${local.watsonx_assistant_url}\""
151154
interpreter = ["/bin/bash", "-c"]
152155
quiet = true
156+
environment = {
157+
IAM_TOKEN = local.sensitive_tokendata
158+
}
153159
}
154160
}
155161

156-
# assistant custom extensions
157-
# manual step - awaiting API
158-
159-
# assistant skills import
160-
# skip for now - depends on manual step for custom extensions
161-
/*
162-
resource "null_resource" "assistant_import_rag_pattern-action-skill" {
163-
triggers = {
164-
always_run = timestamp()
165-
ibmcloud_api_key = var.ibmcloud_api_key
166-
watsonx_assistant_url = local.watsonx_assistant_url
167-
}
168-
169-
provisioner "local-exec" {
170-
command = <<EOF
171-
#!/bin/bash
172-
ASSISTANT_ID=$(curl -X GET -u "apikey:${var.ibmcloud_api_key}" "${local.watsonx_assistant_url}/v2/assistants?version=2023-06-15" \
173-
| jq '.assistants[] | select(.name == "gen-ai-rag-sample-app-assistant") | .assistant_id ')
174-
175-
curl -X POST -u "apikey:${var.ibmcloud_api_key}" --header "Content-Type: application/json" \
176-
--data "@./artifacts/watsonX.Assistant/cc-bank-loan-v1-action.json" \
177-
"${local.watsonx_assistant_url}/v2/assistants/$ASSISTANT_ID/skills_import?version=2023-06-15"
178-
EOF
179-
}
180-
}
181-
*/
182-
183162
# get assistant integration ID
184163
data "external" "assistant_get_integration_id" {
185164
depends_on = [null_resource.assistant_project_creation]
@@ -300,6 +279,7 @@ resource "ibm_cd_tekton_pipeline_trigger" "cd_pipeline_inventory_promotion_trigg
300279

301280
# Trigger webhook to start CI pipeline run
302281
resource "null_resource" "ci_pipeline_run" {
282+
count = var.trigger_ci_pipeline_run == true ? 1 : 0
303283
depends_on = [
304284
ibm_cd_tekton_pipeline_trigger.ci_pipeline_webhook,
305285
ibm_cd_tekton_pipeline_property.watsonx_assistant_integration_id_pipeline_property_ci,

‎solutions/banking/variables.tf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,9 @@ variable "secrets_manager_region" {
110110
description = "The region where the Secrets Manager instance previously created reside."
111111
type = string
112112
}
113+
114+
variable "trigger_ci_pipeline_run" {
115+
description = "Whether to trigger the CI pipeline to build and deploy the application when deploying this solution"
116+
type = bool
117+
default = true
118+
}

‎solutions/banking/version.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ terraform {
1414
}
1515
random = {
1616
source = "hashicorp/random"
17-
version = ">= 3.6.1"
17+
version = ">= 3.5.1"
1818
}
1919
restapi = {
2020
source = "Mastercard/restapi"

‎solutions/banking/watson-scripts/assistant-get-integration-id.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ set -e
55
eval "$(jq -r '@sh "IAM_TOKEN=\(.tokendata) WATSON_ASSISTANT_URL=\(.watson_assistant_url)"')"
66
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
77

8-
ASSISTANT_ID=$(curl -X GET --location "$WATSON_ASSISTANT_URL/v2/assistants?version=2023-06-15" \
8+
ASSISTANT_ID=$(curl -X GET -retry 3 -flsS --location "$WATSON_ASSISTANT_URL/v2/assistants?version=2023-06-15" \
99
--header "Authorization: Bearer $token" \
1010
--header "Content-Type: application/json" \
1111
| jq -r '.assistants[] | select(.name == "gen-ai-rag-sample-app-assistant") | .assistant_id ')
1212

13-
ENVIRONMENT=$(curl -X GET --location "$WATSON_ASSISTANT_URL/v2/assistants/$ASSISTANT_ID/environments?version=2023-06-15" \
13+
ENVIRONMENT=$(curl -X GET -retry 3 -flsS --location "$WATSON_ASSISTANT_URL/v2/assistants/$ASSISTANT_ID/environments?version=2023-06-15" \
1414
--header "Authorization: Bearer $token" \
1515
--header "Content-Type: application/json" \
1616
| jq -r '.environments[] | select(.name == "draft") ')

‎solutions/banking/watson-scripts/assistant-project-creation.sh

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,15 @@
22

33
set -e
44

5-
IAM_TOKEN="$1"
6-
WATSON_ASSISTANT_URL=$2
5+
WATSON_ASSISTANT_URL=$1
76
project_name="gen-ai-rag-sample-app-assistant"
87

8+
# Expects the environment variable $IBMCLOUD_API_KEY to be set
9+
if [[ -z "${IAM_TOKEN}" ]]; then
10+
echo "API key must be set with IAM_TOKEN environment variable" >&2
11+
exit 1
12+
fi
13+
914
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
1015

1116
ASSISTANT_ID=$(curl --retry 3 -fLsS -X GET --location "$WATSON_ASSISTANT_URL/v2/assistants?version=2023-06-15" \
@@ -14,7 +19,7 @@ ASSISTANT_ID=$(curl --retry 3 -fLsS -X GET --location "$WATSON_ASSISTANT_URL/v2/
1419
| jq -r '.assistants[] | select(.name == "'$project_name'") | .assistant_id ')
1520

1621
if [[ -z "$ASSISTANT_ID" ]]; then
17-
# if not ASSISTANT_ID is found then create a new assistant project
22+
# if ASSISTANT_ID is not found then create a new assistant project
1823
curl --retry 3 -fLsS -X POST --location "$WATSON_ASSISTANT_URL/v2/assistants?version=2023-06-15" \
1924
--header "Authorization: Bearer $token" \
2025
--header "Content-Type: application/json" \

‎solutions/banking/watson-scripts/discovery-collection-creation.sh

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,36 @@
11
#!/usr/bin/env bash
22

3-
set -e
3+
set -euo pipefail
44

5-
IAM_TOKEN="$1"
6-
WATSON_DISCOVERY_URL=$2
5+
WATSON_DISCOVERY_URL=$1
6+
DISCOVERY_PROJECT_NAME="gen-ai-rag-sample-app-project"
7+
DISCOVERY_COLLECTION_NAME="gen-ai-rag-sample-app-data"
8+
9+
# Expects the environment variable $IBMCLOUD_API_KEY to be set
10+
if [[ -z "${IAM_TOKEN}" ]]; then
11+
echo "API key must be set with IAM_TOKEN environment variable" >&2
12+
exit 1
13+
fi
714

815
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
916

10-
PROJECT_ID=$(curl -X GET --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
17+
PROJECT_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
1118
--header "Authorization: Bearer $token" \
1219
--header "Content-Type: application/json" \
13-
| jq -r '.projects[] | select(.name == "gen-ai-rag-sample-app-project") | .project_id ')
20+
| jq -r --arg DISCOVERY_PROJECT_NAME "$DISCOVERY_PROJECT_NAME" '.projects[] | select(.name==$DISCOVERY_PROJECT_NAME) | .project_id ')
1421

15-
curl -X POST --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
22+
EXISTING_COLLECTION_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
1623
--header "Authorization: Bearer $token" \
1724
--header "Content-Type: application/json" \
18-
--data "{ \"name\": \"gen-ai-rag-sample-app-data\", \"description\": \"Sample data\" }"
25+
| jq -r --arg DISCOVERY_COLLECTION_NAME "$DISCOVERY_COLLECTION_NAME" '.collections[] | select(.name==$DISCOVERY_COLLECTION_NAME) | .collection_id ')
26+
27+
if [[ -z $EXISTING_COLLECTION_ID ]]; then
28+
curl -X POST -flsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
29+
--header "Authorization: Bearer $token" \
30+
--header "Content-Type: application/json" \
31+
--data "{ \"name\": \"$DISCOVERY_COLLECTION_NAME\", \"description\": \"Sample data\" }"
32+
echo "Collection '$DISCOVERY_COLLECTION_NAME' created"
33+
else
34+
echo "Collection '$DISCOVERY_COLLECTION_NAME' already exists, skipping..."
35+
exit 0
36+
fi

‎solutions/banking/watson-scripts/discovery-file-upload.sh

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,61 @@
22

33
set -e
44

5-
IAM_TOKEN="$1"
6-
WATSON_DISCOVERY_URL="$2"
7-
ARTIFACT_DIRECTORY="$3"
5+
WATSON_DISCOVERY_URL="$1"
6+
ARTIFACT_DIRECTORY="$2"
7+
DISCOVERY_PROJECT_NAME="gen-ai-rag-sample-app-project"
8+
DISCOVERY_COLLECTION_NAME="gen-ai-rag-sample-app-data"
9+
10+
# Expects the environment variable $IBMCLOUD_API_KEY to be set
11+
if [[ -z "${IAM_TOKEN}" ]]; then
12+
echo "API key must be set with IAM_TOKEN environment variable" >&2
13+
exit 1
14+
fi
815

916
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
1017

11-
PROJECT_ID=$(curl -X GET --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
18+
# Get project ID
19+
PROJECT_ID=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
20+
--header "Authorization: Bearer $token" \
21+
--header "Content-Type: application/json" \
22+
| jq -r --arg DISCOVERY_PROJECT_NAME "$DISCOVERY_PROJECT_NAME" '.projects[] | select(.name==$DISCOVERY_PROJECT_NAME) | .project_id ')
23+
24+
# Get collection ID
25+
COLLECTION_ID=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
1226
--header "Authorization: Bearer $token" \
1327
--header "Content-Type: application/json" \
14-
| jq -r '.projects[] | select(.name == "gen-ai-rag-sample-app-project") | .project_id ')
28+
| jq -r --arg DISCOVERY_COLLECTION_NAME "$DISCOVERY_COLLECTION_NAME" '.collections[] | select(.name==$DISCOVERY_COLLECTION_NAME) | .collection_id ')
1529

16-
COLLECTION_ID=$(curl -X GET --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
30+
# Check if documents already exist
31+
EXISTING_DOCUMENTS_ARRAY=()
32+
EXISTING_DOCUMENTS=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents?status=available&version=2023-03-31" \
1733
--header "Authorization: Bearer $token" \
1834
--header "Content-Type: application/json" \
19-
| jq -r '.collections[] | select(.name == "gen-ai-rag-sample-app-data") | .collection_id ')
35+
| jq -r '.documents[] | .document_id' )
36+
EXISTING_DOCUMENTS_ARRAY=("$EXISTING_DOCUMENTS")
37+
EXISTING_DOCUMENT_NAMES=()
38+
39+
if [[ -z "${EXISTING_DOCUMENTS_ARRAY[*]}" ]]; then
40+
echo "Documents list is empty, skipping"
41+
else
42+
for doc in "${EXISTING_DOCUMENTS_ARRAY[@]}"; do
43+
DOCUMENT_NAME=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents/$doc?status=available&version=2023-03-31" \
44+
--header "Authorization: Bearer $token" \
45+
--header "Content-Type: application/json" \
46+
| jq -r '.filename' )
47+
EXISTING_DOCUMENT_NAMES+=("${DOCUMENT_NAME}")
48+
done
49+
fi
2050

51+
# Upload documents if they don't exist in the collection already
2152
for i in {1..7};
2253
do
23-
curl -X POST --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents?version=2023-03-31" \
24-
--header "Authorization: Bearer $token" \
25-
--form "file=@$ARTIFACT_DIRECTORY/FAQ-$i.pdf" \
26-
--form metadata="{\"field_name\": \"text\"}"
54+
if ! [[ ${EXISTING_DOCUMENT_NAMES[*]} =~ FAQ-$i.pdf ]]; then
55+
curl -X POST -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents?version=2023-03-31" \
56+
--header "Authorization: Bearer $token" \
57+
--form "file=@$ARTIFACT_DIRECTORY/FAQ-$i.pdf" \
58+
--form metadata="{\"field_name\": \"text\"}"
59+
else
60+
echo "Document FAQ-$i.pdf already exists, skipping..."
61+
fi
2762
done

‎solutions/banking/watson-scripts/discovery-get-project.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ set -e
55
eval "$(jq -r '@sh "IAM_TOKEN=\(.tokendata) WATSON_DISCOVERY_URL=\(.watson_discovery_url)"')"
66
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
77

8-
PROJECT_ID=$(curl -X GET --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
8+
PROJECT_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
99
--header "Authorization: Bearer $token" \
1010
--header "Content-Type: application/json" \
1111
| jq -r '.projects[] | select(.name == "gen-ai-rag-sample-app-project") | .project_id ')

‎solutions/banking/watson-scripts/discovery-project-creation.sh

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,29 @@
22

33
set -e
44

5-
IAM_TOKEN="$1"
6-
WATSON_DISCOVERY_URL=$2
5+
WATSON_DISCOVERY_URL=$1
6+
DISCOVERY_PROJECT_NAME="gen-ai-rag-sample-app-project"
7+
8+
# Expects the environment variable $IBMCLOUD_API_KEY to be set
9+
if [[ -z "${IAM_TOKEN}" ]]; then
10+
echo "API key must be set with IAM_TOKEN environment variable" >&2
11+
exit 1
12+
fi
713

814
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
915

10-
curl -X POST --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
16+
EXISTING_PROJECT_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
1117
--header "Authorization: Bearer $token" \
1218
--header "Content-Type: application/json" \
13-
--data "{ \"name\": \"gen-ai-rag-sample-app-project\", \"type\": \"document_retrieval\" }"
19+
| jq -r --arg DISCOVERY_PROJECT_NAME "$DISCOVERY_PROJECT_NAME" '.projects[] | select(.name==$DISCOVERY_PROJECT_NAME) | .project_id ')
20+
21+
if [[ -z $EXISTING_PROJECT_ID ]]; then
22+
curl -X POST -flsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
23+
--header "Authorization: Bearer $token" \
24+
--header "Content-Type: application/json" \
25+
--data "{ \"name\": \"$DISCOVERY_PROJECT_NAME\", \"type\": \"document_retrieval\" }"
26+
echo "Project '$DISCOVERY_PROJECT_NAME' created"
27+
else
28+
echo "Project '$DISCOVERY_PROJECT_NAME' already exists, skipping project creation..."
29+
exit 0
30+
fi

0 commit comments

Comments
 (0)