Skip to content

Commit 7342b6d

Browse files
StrahinjaJacimovicStrahinjaJacimovic
authored
Merge pull request #143 from MikroElektronika/improvement/copy-kibana-mikroe
Added workflow for copying indexes to DBP
2 parents 3392b05 + 14aa790 commit 7342b6d

File tree

3 files changed

+35
-11
lines changed

3 files changed

+35
-11
lines changed

‎.github/workflows/checkIndexes.yaml

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ on:
2626
schedule:
2727
- cron: "0/30 7-16 * * 1-5" # Every 30 minutes, between 07:00 AM and 04:59 PM, Monday through Friday
2828

29+
env:
30+
GLOBAL_REGEX: "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
31+
2932
jobs:
3033
manual_run:
3134
if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -51,7 +54,7 @@ jobs:
5154
FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input
5255
LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input
5356
echo "LOG_ONLY is set to $LOG_ONLY"
54-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
57+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
5558
continue-on-error: true # Ensure the workflow continues
5659

5760
- name: Check Indexed Links - Test
@@ -60,7 +63,7 @@ jobs:
6063
FIX_ACTION=${{ github.event.inputs.fix }} # Capture the fix input
6164
LOG_ONLY=$([[ "$FIX_ACTION" == "false" ]] && echo true || echo false) # Negate the fix input
6265
echo "LOG_ONLY is set to $LOG_ONLY"
63-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
66+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "${{ github.event.inputs.regex }}" "--log_only" "$LOG_ONLY"
6467
continue-on-error: true # Ensure the workflow continues
6568

6669
push_to_main_run:
@@ -83,12 +86,12 @@ jobs:
8386
8487
- name: Check Indexed Links - Live
8588
run: |
86-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
89+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
8790
continue-on-error: true # Ensure the workflow continues
8891

8992
- name: Check Indexed Links - Test
9093
run: |
91-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
94+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
9295
continue-on-error: true # Ensure the workflow continues
9396

9497
scheduled_run:
@@ -111,10 +114,10 @@ jobs:
111114
112115
- name: Check Indexed Links - Live
113116
run: |
114-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
117+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_LIVE }} "--es_regex" "$GLOBAL_REGEX"
115118
continue-on-error: true # Ensure the workflow continues
116119

117120
- name: Check Indexed Links - Test
118121
run: |
119-
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "images_sdk|templates|mikrosdk|mcucard|mcu_card|click|mikromedia|pim|queries|sibrain|kit|board"
122+
python -u scripts/check_indexes.py ${{ github.repository }} ${{ secrets.GITHUB_TOKEN }} ${{ secrets.ES_HOST }} ${{ secrets.ES_USER }} ${{ secrets.ES_PASSWORD }} ${{ secrets.ME_ES_HOST }} ${{ secrets.ME_ES_USER }} ${{ secrets.ME_ES_PASSWORD }} ${{ secrets.ES_INDEX_TEST }} "--es_regex" "$GLOBAL_REGEX"
120123
continue-on-error: true # Ensure the workflow continues

‎scripts/check_indexes.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ def str2bool(v):
2222
parser.add_argument("es_host", help="ES instance host value", type=str)
2323
parser.add_argument("es_user", help="ES instance user value", type=str)
2424
parser.add_argument("es_password", help="ES instance password value", type=str)
25+
parser.add_argument("me_es_host", help="MikroE ES instance host value", type=str)
26+
parser.add_argument("me_es_user", help="MikroE ES instance user value", type=str)
27+
parser.add_argument("me_es_password", help="MikroE ES instance password value", type=str)
2528
parser.add_argument("es_index", help="ES instance index value", type=str)
2629
parser.add_argument("--es_regex", help="Regex to use to fetch indexed items", type=str, default=".+")
2730
parser.add_argument("--log_only", help="If True, will not fix broken links, just log them to std out", type=str2bool, default=False)
@@ -33,6 +36,11 @@ def str2bool(v):
3336
index=args.es_index, token=args.gh_token
3437
)
3538

39+
me_es_instance = es.index(
40+
es_host=args.me_es_host, es_user=args.me_es_user, es_password=args.me_es_password,
41+
index=args.es_index, token=args.gh_token
42+
)
43+
3644
gh_instance = gh.repo(args.gh_repo, args.gh_token)
3745

3846
es_instance.fetch(regex=args.es_regex)
@@ -68,5 +76,9 @@ def str2bool(v):
6876
print("%sINFO: Updated \"gh_package_name\" for %s" % (es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
6977
print("%sOK: Asset \"%s\" download link is correct. - %s" % (es_instance.Colors.OKBLUE, indexed_item['source']['name'], indexed_item['source']['download_link']))
7078

79+
# For new elasticsearch DBP it is crucial not to use doc_type for indexing
80+
me_es_instance.update(None, indexed_item['doc']['id'], indexed_item['source'])
81+
print("%sINFO: Copied \"%s\" index from AWS to DBS" % (me_es_instance.Colors.UNDERLINE, indexed_item['source']['name']))
82+
7183
if err and args.log_only:
7284
sys.exit(-1)

‎scripts/classes/class_es.py

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,21 @@ def create(self, doc_type, doc_id, doc_body):
119119

120120
def update(self, doc_type, doc_id, doc_body):
121121
response = self.api_index(self.es_instance, self.index, doc_type, doc_id, doc_body)
122-
if response['created'] and 'created' == response['result']:
123-
print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link']))
124-
elif not 'updated' == response['result']:
125-
raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index))
122+
if doc_type:
123+
if response['created'] and 'created' == response['result']:
124+
print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link']))
125+
elif not 'updated' == response['result']:
126+
raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index))
127+
else:
128+
print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link']))
129+
# For new DBP elasticsearch there is no 'created' field in response, so we just check 'result'
126130
else:
127-
print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link']))
131+
if 'created' == response['result']:
132+
print("%sWARNING: Asset \"%s\" created instead of updating. - %s" % (self.Colors.WARNING, doc_body['name'], doc_body['download_link']))
133+
elif not 'updated' == response['result']:
134+
raise ValueError("%s%s failed to update on %s!" % (self.Colors.FAIL, doc_id, self.index))
135+
else:
136+
print("%sINFO: Asset \"%s\" updated. - %s" % (self.Colors.OKGREEN, doc_body['name'], doc_body['download_link']))
128137

129138
def delete(self, doc_type, doc_id):
130139
response = self.es_instance.delete(

0 commit comments

Comments
 (0)