Skip to content

Commit f35a336

Browse files
in-1911in-1911
authored
feat: Elastic configuration for Watson Assistant (#120)
- Control the provisioning of different Watson sub-components - Watson Discovery: if `watson_discovery_instance_id` is provided, a discovery project with a collection will be created and sample data from [artifacts/WatsonDiscovery](/terraform-ibm-modules/terraform-ibm-rag-sample-da/tree/main/solutions/banking/artifacts/WatsonDiscovery) will be uploaded into a collection. Previous version always created Discovery project and collection. - Watson Machine Learning: if `watson_machine_learning_instance_guid` is provided, a COS instance and a project referencing it will be created in the WML instance. Previous version always created WML project and COS instance. - Elastic index: if `elastic_instance_crn` is provided, the following resources will be provisioned: - New index in the Elastic DB - Sample data from [bank loan FAQs](/terraform-ibm-modules/terraform-ibm-rag-sample-da/tree/main/solutions/banking/artifacts/watsonx.Assistant/bank-loan-faqs.json) uploaded into the index (can be skipped if `elastic_upload_sample_data = false` ) - Search skill enabled in Watson Assistant workspace pointed to the Elastic index with specified credentials (needs Elastic service credentials `wxasst_db_user`) - Action skill enabled in Watson Assistant workspace using [predefined action](/terraform-ibm-modules/terraform-ibm-rag-sample-da/tree/main/solutions/banking/artifacts/watsonx.Assistant/wxa-conv-srch-es-v1.json). - When Elastic index option is used, this DA results in fully configured Assistant that can be immediately used with the application. - Dependencies on CD instance added for better handling of provisioning / destroying when a new CD instance is created - otherwise pipeline resource creation may fail if pipelines are older than a grace period.
1 parent c900cba commit f35a336

33 files changed

+2129
-137
lines changed

‎.secrets.baseline

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"files": "go.sum|^.secrets.baseline$",
44
"lines": null
55
},
6-
"generated_at": "2024-05-31T17:16:48Z",
6+
"generated_at": "2024-07-19T17:07:55Z",
77
"plugins_used": [
88
{
99
"name": "AWSKeyDetector"
@@ -77,50 +77,60 @@
7777
}
7878
],
7979
"results": {
80-
"solutions/banking/artifacts/watsonx.Assistant/cc-bank-loan-v1-action.json": [
80+
"modules/watson-assistant/scripts/assistant-create.sh": [
8181
{
82-
"hashed_secret": "859b9d6bf67cb6fdac5cc9dca1fbfd11191d26f6",
82+
"hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
8383
"is_secret": false,
8484
"is_verified": false,
85-
"line_number": 913,
86-
"type": "Hex High Entropy String",
85+
"line_number": 4,
86+
"type": "Secret Keyword",
8787
"verified_result": null
88-
},
88+
}
89+
],
90+
"modules/watson-assistant/scripts/assistant-destroy.sh": [
8991
{
90-
"hashed_secret": "1020bf59dff762463413f6c7a536c4e5254d1dd8",
92+
"hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
9193
"is_secret": false,
9294
"is_verified": false,
93-
"line_number": 1301,
94-
"type": "Hex High Entropy String",
95+
"line_number": 5,
96+
"type": "Secret Keyword",
9597
"verified_result": null
9698
}
9799
],
98-
"solutions/banking/watson-scripts/assistant-create.sh": [
100+
"modules/watson-assistant/scripts/assistant-read.sh": [
99101
{
100102
"hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
101103
"is_secret": false,
102104
"is_verified": false,
103-
"line_number": 4,
105+
"line_number": 5,
104106
"type": "Secret Keyword",
105107
"verified_result": null
106108
}
107109
],
108-
"solutions/banking/watson-scripts/assistant-destroy.sh": [
110+
"solutions/banking/artifacts/watsonx.Assistant/cc-bank-loan-v1-action.json": [
109111
{
110-
"hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
112+
"hashed_secret": "859b9d6bf67cb6fdac5cc9dca1fbfd11191d26f6",
111113
"is_secret": false,
112114
"is_verified": false,
113-
"line_number": 5,
114-
"type": "Secret Keyword",
115+
"line_number": 913,
116+
"type": "Hex High Entropy String",
117+
"verified_result": null
118+
},
119+
{
120+
"hashed_secret": "1020bf59dff762463413f6c7a536c4e5254d1dd8",
121+
"is_secret": false,
122+
"is_verified": false,
123+
"line_number": 1301,
124+
"type": "Hex High Entropy String",
115125
"verified_result": null
116126
}
117127
],
118-
"solutions/banking/watson-scripts/assistant-read.sh": [
128+
"solutions/banking/artifacts/watsonx.Assistant/elastic-search-skill.json": [
119129
{
120-
"hashed_secret": "20e0438047329f2ff165b6d2a4c081e42d280a3b",
130+
"hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
121131
"is_secret": false,
122132
"is_verified": false,
123-
"line_number": 5,
133+
"line_number": 17,
124134
"type": "Secret Keyword",
125135
"verified_result": null
126136
}

‎modules/elastic-index/main.tf

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
locals {
2+
elastic_auth_base64 = sensitive(base64encode("${var.elastic_service_binding.username}:${var.elastic_service_binding.password}"))
3+
}
4+
5+
# Elastic index creation
6+
7+
resource "elasticstack_elasticsearch_index" "sample_index" {
8+
name = var.elastic_index_name
9+
deletion_protection = false
10+
}
11+
12+
## Upload sample data entries
13+
14+
/*
15+
# Note: Elastic uses CA certificate for connection, and restapi does not have an option to specify that
16+
# Therefore the Elastic APIs are used from shell_script
17+
*/
18+
19+
resource "shell_script" "elastic_index_entries" {
20+
count = var.elastic_index_entries_file != null ? 1 : 0
21+
depends_on = [elasticstack_elasticsearch_index.sample_index] # shell_script.elastic_index
22+
lifecycle_commands {
23+
create = <<-EOT
24+
set -e
25+
source ./${path.module}/scripts/elastic-index-utils.sh
26+
create_index_entries
27+
EOT
28+
delete = <<-EOT
29+
set -e
30+
source ./${path.module}/scripts/elastic-index-utils.sh
31+
delete_index_entries
32+
EOT
33+
}
34+
35+
environment = {
36+
ELASTIC_URL = var.elastic_service_binding.url
37+
ELASTIC_CACERT = var.elastic_service_binding.ca_data_base64
38+
ELASTIC_INDEX_NAME = urlencode(elasticstack_elasticsearch_index.sample_index.name) # shell_script.elastic_index.output.index_name)
39+
ELASTIC_ENTRIES_FILE = var.elastic_index_entries_file
40+
MODULE_PATH = path.module
41+
}
42+
43+
sensitive_environment = {
44+
ELASTIC_AUTH_BASE64 = local.elastic_auth_base64
45+
}
46+
47+
# Change in apikey should not trigger assistant re-create
48+
lifecycle {
49+
ignore_changes = [sensitive_environment]
50+
}
51+
}

‎modules/elastic-index/outputs.tf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
output "elastic_index" {
2+
description = "Elastic DB index attributes."
3+
value = elasticstack_elasticsearch_index.sample_index
4+
}
5+
6+
output "elastic_upload_count" {
7+
description = "Count of uploaded entries."
8+
value = var.elastic_index_entries_file != null ? tonumber(shell_script.elastic_index_entries[0].output.count) : 0
9+
}

‎modules/elastic-index/scripts/elastic-index-utils.sh

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/usr/bin/env bash
2+
3+
function create_index() {
4+
local OUTPUT
5+
# Skip -f
6+
OUTPUT=$(curl -LsS -X PUT --location "$ELASTIC_URL/$ELASTIC_INDEX_NAME" \
7+
--header "Authorization: Basic $ELASTIC_AUTH_BASE64" \
8+
--header "Accepts: application/json" \
9+
--cacert <(cat <<< "$ELASTIC_CACERT" | base64 -d) )
10+
check_error "$OUTPUT"
11+
# Read index attributes
12+
get_index "$ELASTIC_INDEX_NAME"
13+
}
14+
15+
function read_index() {
16+
IN=$(cat)
17+
local existing_index_name
18+
existing_index_name="$(echo "$IN" | jq -r '.index_name')"
19+
if [[ -z "$existing_index_name" ]]; then
20+
exit 0
21+
fi
22+
get_index "$existing_index_name"
23+
}
24+
25+
function get_index() {
26+
local existing_index_name
27+
existing_index_name=$1
28+
local OUTPUT
29+
OUTPUT=$(curl -LsS -X GET --location "${ELASTIC_URL}/$existing_index_name" \
30+
--header "Authorization: Basic $ELASTIC_AUTH_BASE64" \
31+
--cacert <(cat <<< "$ELASTIC_CACERT" | base64 -d) \
32+
--header "Accepts: application/json" )
33+
check_error "$OUTPUT"
34+
echo "$OUTPUT" | jq '. | to_entries[] | select( .value.aliases != null )' | jq -S --arg instance_id "$ELASTIC_INSTANCE_ID" '{"instance_id": $instance_id, "index_name": .key, "id": .value.settings.index.uuid}'
35+
}
36+
37+
38+
function delete_index() {
39+
IN=$(cat)
40+
local existing_index_name
41+
existing_index_name=$(echo "$IN" | jq -r '.index_name')
42+
if [[ -z "$existing_index_name" ]]; then
43+
exit 0
44+
fi
45+
local OUTPUT
46+
OUTPUT=$(curl -LsS -X DELETE --location "$ELASTIC_URL/$existing_index_name" \
47+
--header "Authorization: Basic $ELASTIC_AUTH_BASE64" \
48+
--header "Accepts: application/json" \
49+
--cacert <(cat <<< "$ELASTIC_CACERT" | base64 -d) )
50+
check_error "$OUTPUT"
51+
echo "$OUTPUT" | jq '. | select( .acknowledged == true )'
52+
}
53+
54+
function create_index_entries() {
55+
local OUTPUT
56+
57+
OUTPUT=$(curl -LsS -X POST --location "$ELASTIC_URL/$ELASTIC_INDEX_NAME/_bulk" \
58+
--header "Authorization: Basic $ELASTIC_AUTH_BASE64" \
59+
--header "Content-Type: application/json" \
60+
--header "Accepts: application/json" \
61+
--data-binary @<(jq -r '.[] | tojson | ("{\"index\" : {} }\n" + .)' < "$ELASTIC_ENTRIES_FILE") \
62+
--cacert <(cat <<< "$ELASTIC_CACERT" | base64 -d)
63+
)
64+
check_error "$OUTPUT"
65+
if [[ $(echo "$OUTPUT" | jq -r '. | .errors') == "true" ]]; then
66+
echo "$OUTPUT" | jq '[.items[] | select( .index?.error != null ) | .index]'
67+
exit 1
68+
fi
69+
echo "$OUTPUT" | jq '{ "count": (.items | length) }'
70+
}
71+
72+
function delete_index_entries() {
73+
local OUTPUT
74+
75+
OUTPUT=$(curl -LsS -X POST --location "$ELASTIC_URL/$ELASTIC_INDEX_NAME/_delete_by_query?conflicts=proceed" \
76+
--header "Authorization: Basic $ELASTIC_AUTH_BASE64" \
77+
--header "Content-Type: application/json" \
78+
--header "Accepts: application/json" \
79+
--data '{"query": {"match_all": {}}}' \
80+
--cacert <(cat <<< "$ELASTIC_CACERT" | base64 -d)
81+
)
82+
check_error "$OUTPUT"
83+
if [[ ! $(echo "$OUTPUT" | jq -r '. | .failures | length') == "0" ]]; then
84+
echo "$OUTPUT"
85+
exit 1
86+
fi
87+
}
88+
89+
function check_error() {
90+
status=$?
91+
if [ $status -ne 0 ]; then
92+
echo "API call returned an error code $status"
93+
echo "$1"
94+
exit 1
95+
fi
96+
97+
local error
98+
error=$(echo "$1" | jq '.error')
99+
if [[ -n "$error" && ! "$error" == "null" ]]; then
100+
echo "$error" | jq
101+
exit 1
102+
fi
103+
}

‎modules/elastic-index/variables.tf

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
variable "elastic_service_binding" {
2+
description = "Elastic ICD instance credentials"
3+
type = object({
4+
url = string
5+
username = string
6+
password = string
7+
ca_data_base64 = optional(string)
8+
})
9+
}
10+
11+
variable "elastic_index_name" {
12+
description = "Name of index in Elastic instance"
13+
type = string
14+
default = "sample-rag-app-content"
15+
}
16+
17+
variable "elastic_index_entries_file" {
18+
description = "Path to JSON file with content entries"
19+
type = string
20+
default = null
21+
}

‎modules/elastic-index/version.tf

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
terraform {
2+
required_providers {
3+
shell = {
4+
source = "scottwinkler/shell"
5+
version = ">= 1.7.10"
6+
}
7+
elasticstack = {
8+
source = "elastic/elasticstack"
9+
version = ">= 0.11.0"
10+
}
11+
}
12+
required_version = ">= 1.3.0"
13+
}

0 commit comments

Comments
 (0)