Skip to content

Commit 1f9a731

Browse files
brendankellyibmbrendankellyibm
authored
feat: add discovery destroy logic (#54)
* added restapi objects for discovery project and collection, tweaked file upload script * add ignore for shellcheck sc2206
1 parent d213c6d commit 1f9a731

File tree

5 files changed

+65
-137
lines changed

5 files changed

+65
-137
lines changed

‎solutions/banking/main.tf

Lines changed: 53 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
locals {
22
watsonx_assistant_url = "https://api.${var.watson_assistant_region}.assistant.watson.cloud.ibm.com/instances/${var.watson_assistant_instance_id}"
3-
watsonx_discovery_url = "https://api.${var.watson_discovery_region}.discovery.watson.cloud.ibm.com/instances/${var.watson_discovery_instance_id}"
3+
watsonx_discovery_url = "//api.${var.watson_discovery_region}.discovery.watson.cloud.ibm.com/instances/${var.watson_discovery_instance_id}"
44
sensitive_tokendata = sensitive(data.ibm_iam_auth_token.tokendata.iam_access_token)
55
}
66

@@ -91,52 +91,69 @@ module "configure_project" {
9191
cos_crn = module.cos.cos_instance_crn
9292
}
9393

94-
# get zip file from code repo
95-
# add deployment space - not for demo scope
96-
97-
# discovery project creation
98-
# possibly change type of project here - TBC
99-
resource "null_resource" "discovery_project_creation" {
100-
triggers = {
101-
always_run = timestamp()
102-
}
103-
104-
provisioner "local-exec" {
105-
command = "${path.module}/watson-scripts/discovery-project-creation.sh \"${local.watsonx_discovery_url}\""
106-
interpreter = ["/bin/bash", "-c"]
107-
quiet = true
108-
environment = {
109-
IAM_TOKEN = local.sensitive_tokendata
110-
}
111-
}
94+
# Discovery project creation
95+
resource "restapi_object" "configure_discovery_project" {
96+
depends_on = [data.ibm_iam_auth_token.tokendata]
97+
path = local.watsonx_discovery_url
98+
read_path = "${local.watsonx_discovery_url}/v2/projects/{id}?version=2023-03-31"
99+
read_method = "GET"
100+
create_path = "${local.watsonx_discovery_url}/v2/projects?version=2023-03-31"
101+
create_method = "POST"
102+
id_attribute = "project_id"
103+
destroy_method = "DELETE"
104+
destroy_path = "${local.watsonx_discovery_url}/v2/projects/{id}?version=2023-03-31"
105+
data = <<-EOT
106+
{
107+
"name": "gen-ai-rag-sample-app-project",
108+
"type": "document_retrieval"
109+
}
110+
EOT
111+
update_method = "POST"
112+
update_path = "${local.watsonx_discovery_url}/v2/projects/{id}?version=2023-03-31"
113+
update_data = <<-EOT
114+
{
115+
"name": "gen-ai-rag-sample-app-project",
116+
"type": "document_retrieval"
117+
}
118+
EOT
112119
}
113120

114-
# discovery collection creation
115-
resource "null_resource" "discovery_collection_creation" {
116-
depends_on = [null_resource.discovery_project_creation]
117-
triggers = {
118-
always_run = timestamp()
119-
}
120-
121-
provisioner "local-exec" {
122-
command = "${path.module}/watson-scripts/discovery-collection-creation.sh \"${local.watsonx_discovery_url}\""
123-
interpreter = ["/bin/bash", "-c"]
124-
quiet = true
125-
environment = {
126-
IAM_TOKEN = local.sensitive_tokendata
127-
}
128-
}
121+
# Discovery collection creation
122+
resource "restapi_object" "configure_discovery_collection" {
123+
depends_on = [data.ibm_iam_auth_token.tokendata, restapi_object.configure_discovery_project]
124+
path = local.watsonx_discovery_url
125+
read_path = "${local.watsonx_discovery_url}/v2/projects/${restapi_object.configure_discovery_project.id}/collections/{id}?version=2023-03-31"
126+
read_method = "GET"
127+
create_path = "${local.watsonx_discovery_url}/v2/projects/${restapi_object.configure_discovery_project.id}/collections?version=2023-03-31"
128+
create_method = "POST"
129+
id_attribute = "collection_id"
130+
destroy_method = "DELETE"
131+
destroy_path = "${local.watsonx_discovery_url}/v2/projects/${restapi_object.configure_discovery_project.id}/collections/{id}?version=2023-03-31"
132+
data = <<-EOT
133+
{
134+
"name": "gen-ai-rag-sample-app-data",
135+
"description": "Sample data"
136+
}
137+
EOT
138+
update_method = "POST"
139+
update_path = "${local.watsonx_discovery_url}/v2/projects/${restapi_object.configure_discovery_project.id}/collections/{id}?version=2023-03-31"
140+
update_data = <<-EOT
141+
{
142+
"name": "gen-ai-rag-sample-app-data",
143+
"description": "Sample data"
144+
}
145+
EOT
129146
}
130147

131148
# discovery file upload
132149
resource "null_resource" "discovery_file_upload" {
133-
depends_on = [null_resource.discovery_collection_creation]
150+
depends_on = [restapi_object.configure_discovery_collection]
134151
triggers = {
135152
always_run = timestamp()
136153
}
137154

138155
provisioner "local-exec" {
139-
command = "${path.module}/watson-scripts/discovery-file-upload.sh \"${local.watsonx_discovery_url}\" \"${path.module}/artifacts/WatsonDiscovery\" "
156+
command = "${path.module}/watson-scripts/discovery-file-upload.sh \"https:${local.watsonx_discovery_url}\" \"${restapi_object.configure_discovery_project.id}\" \"${restapi_object.configure_discovery_collection.id}\" \"${path.module}/artifacts/WatsonDiscovery\" "
140157
interpreter = ["/bin/bash", "-c"]
141158
quiet = true
142159
environment = {
@@ -171,17 +188,6 @@ data "external" "assistant_get_integration_id" {
171188
}
172189
}
173190

174-
# get discovery project ID
175-
data "external" "discovery_project_id" {
176-
depends_on = [null_resource.discovery_project_creation]
177-
program = ["bash", "${path.module}/watson-scripts/discovery-get-project.sh"]
178-
query = {
179-
tokendata = local.sensitive_tokendata
180-
watson_discovery_url = local.watsonx_discovery_url
181-
}
182-
}
183-
184-
185191
# Update CI pipeline with Assistant instance ID
186192
resource "ibm_cd_tekton_pipeline_property" "watsonx_assistant_id_pipeline_property_ci" {
187193
provider = ibm.ibm_resources

‎solutions/banking/outputs.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,5 @@ output "watsonx_assistant_integration_id" {
3030

3131
output "watson_discovery_project_id" {
3232
description = "Watson Discovery Project ID."
33-
value = data.external.discovery_project_id.result.discovery_project_id
33+
value = restapi_object.configure_discovery_project.id
3434
}

‎solutions/banking/watson-scripts/discovery-collection-creation.sh

Lines changed: 0 additions & 36 deletions
This file was deleted.

‎solutions/banking/watson-scripts/discovery-file-upload.sh

Lines changed: 11 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33
set -e
44

55
WATSON_DISCOVERY_URL="$1"
6-
ARTIFACT_DIRECTORY="$2"
7-
DISCOVERY_PROJECT_NAME="gen-ai-rag-sample-app-project"
8-
DISCOVERY_COLLECTION_NAME="gen-ai-rag-sample-app-data"
6+
WATSON_DISCOVERY_PROJECT_ID="$2"
7+
WATSON_DISCOVERY_COLLECTION_ID="$3"
8+
ARTIFACT_DIRECTORY="$4"
99

1010
# Expects the environment variable $IBMCLOUD_API_KEY to be set
1111
if [[ -z "${IAM_TOKEN}" ]]; then
@@ -15,32 +15,20 @@ fi
1515

1616
token="$(echo "$IAM_TOKEN" | awk '{print $2}')"
1717

18-
# Get project ID
19-
PROJECT_ID=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects?version=2023-03-31" \
20-
--header "Authorization: Bearer $token" \
21-
--header "Content-Type: application/json" \
22-
| jq -r --arg DISCOVERY_PROJECT_NAME "$DISCOVERY_PROJECT_NAME" '.projects[] | select(.name==$DISCOVERY_PROJECT_NAME) | .project_id ')
23-
24-
# Get collection ID
25-
COLLECTION_ID=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections?version=2023-03-31" \
26-
--header "Authorization: Bearer $token" \
27-
--header "Content-Type: application/json" \
28-
| jq -r --arg DISCOVERY_COLLECTION_NAME "$DISCOVERY_COLLECTION_NAME" '.collections[] | select(.name==$DISCOVERY_COLLECTION_NAME) | .collection_id ')
29-
3018
# Check if documents already exist
31-
EXISTING_DOCUMENTS_ARRAY=()
32-
EXISTING_DOCUMENTS=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents?status=available&version=2023-03-31" \
19+
DISCOVERY_QUERY_RESULTS=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$WATSON_DISCOVERY_PROJECT_ID/collections/$WATSON_DISCOVERY_COLLECTION_ID/documents?status=available&version=2023-03-31" \
3320
--header "Authorization: Bearer $token" \
3421
--header "Content-Type: application/json" \
3522
| jq -r '.documents[] | .document_id' )
36-
EXISTING_DOCUMENTS_ARRAY=("$EXISTING_DOCUMENTS")
37-
EXISTING_DOCUMENT_NAMES=()
3823

39-
if [[ -z "${EXISTING_DOCUMENTS_ARRAY[*]}" ]]; then
24+
# shellcheck disable=SC2206
25+
EXISTING_DOCUMENTS=($DISCOVERY_QUERY_RESULTS)
26+
27+
if [[ ${#EXISTING_DOCUMENTS[@]} -eq 0 ]]; then
4028
echo "Documents list is empty, skipping"
4129
else
42-
for doc in "${EXISTING_DOCUMENTS_ARRAY[@]}"; do
43-
DOCUMENT_NAME=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents/$doc?status=available&version=2023-03-31" \
30+
for doc in "${EXISTING_DOCUMENTS[@]}"; do
31+
DOCUMENT_NAME=$(curl -X GET --retry 3 -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$WATSON_DISCOVERY_PROJECT_ID/collections/$WATSON_DISCOVERY_COLLECTION_ID/documents/$doc?status=available&version=2023-03-31" \
4432
--header "Authorization: Bearer $token" \
4533
--header "Content-Type: application/json" \
4634
| jq -r '.filename' )
@@ -52,7 +40,7 @@ fi
5240
for i in {1..7};
5341
do
5442
if ! [[ ${EXISTING_DOCUMENT_NAMES[*]} =~ FAQ-$i.pdf ]]; then
55-
curl -X POST -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$PROJECT_ID/collections/$COLLECTION_ID/documents?version=2023-03-31" \
43+
curl -X POST -fLsS --location "$WATSON_DISCOVERY_URL/v2/projects/$WATSON_DISCOVERY_PROJECT_ID/collections/$WATSON_DISCOVERY_COLLECTION_ID/documents?version=2023-03-31" \
5644
--header "Authorization: Bearer $token" \
5745
--form "file=@$ARTIFACT_DIRECTORY/FAQ-$i.pdf" \
5846
--form metadata="{\"field_name\": \"text\"}"

‎solutions/banking/watson-scripts/discovery-project-creation.sh

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)