fix: Update Watson Assistant code to allow for destruction of resource (#85)

BREAKING CHANGE: In this release, the code to manage Watson Assistant was updated to allow for deletion of the Assistant when the RAG sample DA is undeployed. As this is a different mechanism, when upgrading the DA to this or later versions, users may end up with a duplicate Assistant within their Watson Assistant instance. This should not incur additional costs as it will be unused, but we recommend that users navigate to their Watson Assistant instance after the upgrade, and delete the original Assistant from there. Prefixes for assistants are now available, so we recommend using a prefix to generate a unique name for the new assistant.

Author: brendankellyibm

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "go.sum|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-04-04T16:29:33Z",
+  "generated_at": "2024-05-31T17:16:48Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -95,12 +95,32 @@
         "verified_result": null
       }
     ],
-    "solutions/banking/artifacts/watsonx.Assistant/watsonx-custom-ext-api-call.openapi.json": [
+    "solutions/banking/watson-scripts/assistant-create.sh": [
       {
-        "hashed_secret": "2308d0fb5ab83a7e92e8000f5f094342d3f87bd0",
+        "hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 21,
+        "line_number": 4,
+        "type": "Secret Keyword",
+        "verified_result": null
+      }
+    ],
+    "solutions/banking/watson-scripts/assistant-destroy.sh": [
+      {
+        "hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 5,
+        "type": "Secret Keyword",
+        "verified_result": null
+      }
+    ],
+    "solutions/banking/watson-scripts/assistant-read.sh": [
+      {
+        "hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 5,
         "type": "Secret Keyword",
         "verified_result": null
       }

solutions/banking/main.tf

@@ -183,28 +183,23 @@ resource "null_resource" "discovery_file_upload" {
 }
 
 # assistant creation
-resource "null_resource" "assistant_project_creation" {
-  triggers = {
-    always_run = timestamp()
+resource "shell_script" "watson_assistant" {
+  lifecycle_commands {
+    create = file("${path.module}/watson-scripts/assistant-create.sh")
+    delete = file("${path.module}/watson-scripts/assistant-destroy.sh")
+    read   = file("${path.module}/watson-scripts/assistant-read.sh")
   }
 
-  provisioner "local-exec" {
-    command     = "${path.module}/watson-scripts/assistant-project-creation.sh \"${local.watsonx_assistant_url}\""
-    interpreter = ["/bin/bash", "-c"]
-    quiet       = true
-    environment = {
-      IAM_TOKEN = local.sensitive_tokendata
-    }
+  environment = {
+    WATSON_ASSISTANT_API_VERSION = "2023-06-15"
+    WATSON_ASSISTANT_DESCRIPTION = "Generative AI sample app assistant"
+    WATSON_ASSISTANT_LANGUAGE    = "en"
+    WATSON_ASSISTANT_NAME        = var.prefix != "" ? "${var.prefix}-gen-ai-rag-sample-app-assistant" : "gen-ai-rag-sample-app-assistant"
+    WATSON_ASSISTANT_URL         = local.watsonx_assistant_url
   }
-}
 
-# get assistant integration ID
-data "external" "assistant_get_integration_id" {
-  depends_on = [null_resource.assistant_project_creation]
-  program    = ["bash", "${path.module}/watson-scripts/assistant-get-integration-id.sh"]
-  query = {
-    tokendata            = local.sensitive_tokendata
-    watson_assistant_url = local.watsonx_assistant_url
+  sensitive_environment = {
+    IBMCLOUD_API_KEY = var.ibmcloud_api_key
   }
 }
 
@@ -247,21 +242,21 @@ resource "ibm_cd_tekton_pipeline_property" "application_flavor_pipeline_property
 # Update CI pipeline with Assistant integration ID
 resource "ibm_cd_tekton_pipeline_property" "watsonx_assistant_integration_id_pipeline_property_ci" {
   provider    = ibm.ibm_resources
-  depends_on  = [data.external.assistant_get_integration_id]
+  depends_on  = [shell_script.watson_assistant]
   name        = "watsonx_assistant_integration_id"
   pipeline_id = var.ci_pipeline_id
   type        = "text"
-  value       = data.external.assistant_get_integration_id.result.assistant_integration_id
+  value       = shell_script.watson_assistant.output["assistant_integration_id"]
 }
 
 # Update CD pipeline with Assistant integration ID
 resource "ibm_cd_tekton_pipeline_property" "watsonx_assistant_integration_id_pipeline_property_cd" {
   provider    = ibm.ibm_resources
-  depends_on  = [data.external.assistant_get_integration_id]
+  depends_on  = [shell_script.watson_assistant]
   name        = "watsonx_assistant_integration_id"
   pipeline_id = var.cd_pipeline_id
   type        = "text"
-  value       = data.external.assistant_get_integration_id.result.assistant_integration_id
+  value       = shell_script.watson_assistant.output["assistant_integration_id"]
 }
 
 # Random string for webhook token
@@ -289,6 +284,7 @@ resource "ibm_cd_tekton_pipeline_trigger" "ci_pipeline_webhook" {
 
 # Ensure webhook trigger runs against correct git branch
 resource "ibm_cd_tekton_pipeline_trigger_property" "ci_pipeline_webhook_branch_property" {
+  provider    = ibm.ibm_resources
   depends_on  = [ibm_cd_tekton_pipeline_trigger.ci_pipeline_webhook]
   name        = "branch"
   pipeline_id = var.ci_pipeline_id

solutions/banking/outputs.tf

@@ -25,7 +25,7 @@ output "cos_instance_crn" {
 
 output "watsonx_assistant_integration_id" {
   description = "WatsonX assistant integration ID."
-  value       = data.external.assistant_get_integration_id.result.assistant_integration_id
+  value       = shell_script.watson_assistant.output["assistant_integration_id"]
 }
 
 output "watson_discovery_project_id" {

solutions/banking/version.tf

@@ -1,9 +1,5 @@
 terraform {
   required_providers {
-    external = {
-      source  = "hashicorp/external"
-      version = ">=2.3.3"
-    }
     ibm = {
       source  = "IBM-Cloud/ibm"
       version = ">= 1.66.0"
@@ -20,6 +16,10 @@ terraform {
       source  = "Mastercard/restapi"
       version = ">= 1.19.1"
     }
+    shell = {
+      source  = "scottwinkler/shell"
+      version = ">= 1.7.7"
+    }
   }
   required_version = ">= 1.3.0"
 }

solutions/banking/watson-scripts/assistant-create.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+token=$(curl -fLsS -X POST 'https://iam.cloud.ibm.com/identity/token' -H 'Content-Type: application/x-www-form-urlencoded' -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=$IBMCLOUD_API_KEY" | jq -r '.access_token')
+
+OUTPUT=$(curl -fLsS -X POST --location "$WATSON_ASSISTANT_URL/v2/assistants?version=$WATSON_ASSISTANT_API_VERSION" \
+    --header "Authorization: Bearer $token" \
+    --header "Content-Type: application/json" \
+    --data "{\"name\":\"$WATSON_ASSISTANT_NAME\",\"language\":\"$WATSON_ASSISTANT_LANGUAGE\",\"description\":\"$WATSON_ASSISTANT_DESCRIPTION\"}")
+
+ASSISTANT_ID=$(echo "$OUTPUT" | jq -r '.assistant_id')
+ENVIRONMENT_ID=$(echo "$OUTPUT" | jq -r '.assistant_environments[] | select(.name == "draft") | .environment_id')
+
+INTEGRATION_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_ASSISTANT_URL/v2/assistants/$ASSISTANT_ID/environments/$ENVIRONMENT_ID?version=2023-06-15" \
+    --header "Authorization: Bearer $token" \
+    --header "Content-Type: application/json" \
+    | jq -r '.integration_references[] | select(.type == "web_chat") | .integration_id ')
+
+jq -nS --arg assistant_id "$ASSISTANT_ID" --arg assistant_integration_id "$INTEGRATION_ID" '{"assistant_integration_id": $assistant_integration_id, "assistant_id": $assistant_id}'

solutions/banking/watson-scripts/assistant-destroy.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+token=$(curl -fLsS -X POST 'https://iam.cloud.ibm.com/identity/token' -H 'Content-Type: application/x-www-form-urlencoded' -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=$IBMCLOUD_API_KEY" | jq -r '.access_token')
+
+IN=$(cat)
+EXISTING_ASSISTANT_ID=$(echo "$IN" | jq -r .assistant_id)
+
+OUTPUT=$(curl -flsS --retry 3 -X GET --location "$WATSON_ASSISTANT_URL/v2/assistants?version=$WATSON_ASSISTANT_API_VERSION" \
+  --header "Authorization: Bearer $token" \
+  --header "Content-Type: application/json" | jq -r '.assistants[] | select(.assistant_id == "'"$EXISTING_ASSISTANT_ID"'")')
+
+if [[ $OUTPUT ]]; then
+  curl -fLsS -X DELETE --location "$WATSON_ASSISTANT_URL/v2/assistants/$EXISTING_ASSISTANT_ID?version=$WATSON_ASSISTANT_API_VERSION" \
+      --header "Authorization: Bearer $token"
+fi

solutions/banking/watson-scripts/assistant-get-integration-id.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+token=$(curl -fLsS -X POST 'https://iam.cloud.ibm.com/identity/token' -H 'Content-Type: application/x-www-form-urlencoded' -d "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=$IBMCLOUD_API_KEY" | jq -r '.access_token')
+
+IN=$(cat)
+EXISTING_ASSISTANT_ID=$(echo "$IN" | jq -r .assistant_id)
+
+OUTPUT=$(curl -X GET --retry 3 -fLsS --location "$WATSON_ASSISTANT_URL/v2/assistants?version=$WATSON_ASSISTANT_API_VERSION" \
+  --header "Authorization: Bearer $token" \
+  --header "Content-Type: application/json" | jq -r '.assistants[] | select(.assistant_id == "'"$EXISTING_ASSISTANT_ID"'")')
+
+if [ -z "${OUTPUT}" ]; then
+  exit 1
+fi
+
+ASSISTANT_ID=$(echo "$OUTPUT" | jq -r '.assistant_id')
+ENVIRONMENT_ID=$(echo "$OUTPUT" | jq -r '.assistant_environments[] | select(.environment == "draft") | .environment_id')
+
+INTEGRATION_ID=$(curl -X GET --retry 3 -flsS --location "$WATSON_ASSISTANT_URL/v2/assistants/$ASSISTANT_ID/environments/$ENVIRONMENT_ID?version=$WATSON_ASSISTANT_API_VERSION" \
+    --header "Authorization: Bearer $token" \
+    --header "Content-Type: application/json" \
+    | jq -r '.integration_references[] | select(.type == "web_chat") | .integration_id ')
+
+jq -nS --arg assistant_id "$ASSISTANT_ID" --arg assistant_integration_id "$INTEGRATION_ID" '{"assistant_integration_id": $assistant_integration_id, "assistant_id": $assistant_id}'