Skip to content

Commit 2a804ef

Browse files
abuzarmahmoodabuzarmahmood
authored
Merge pull request #224 from abuzarmahmood/8-add-testing-and-cicd-for-repo
8 add testing and cicd for repo
2 parents ee0b8ec + eb5754d commit 2a804ef

File tree

2 files changed

+180
-60
lines changed

2 files changed

+180
-60
lines changed

‎.github/workflows/python_workflow_test.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Description: This is a test workflow for running a Python script on a self-hosted runner
2+
# Don't chain jobs together as any failure in one job will stop the workflow
3+
4+
name: Python test
5+
run-name: python_test
6+
on: [pull_request]
7+
jobs:
8+
Preamble:
9+
runs-on: self-hosted
10+
steps:
11+
- run: cd ~/Desktop/blech_clust
12+
- run: pwd
13+
- run: which python
14+
- run: conda info --envs
15+
- name: Set up repo
16+
uses: actions/checkout@v4
17+
- run: echo "${{ github.ref }} | ${{ github.repository }} | ${{ github.event.pull_request.title }}"
18+
Spike-Only:
19+
runs-on: self-hosted
20+
needs: Preamble
21+
steps:
22+
- name: Set up repo
23+
uses: actions/checkout@v4
24+
- name: Prefect SPIKE only test
25+
shell: bash
26+
working-directory: /home/exouser/Desktop/blech_clust
27+
run: python pipeline_testing/prefect_pipeline.py -s 2>&1 |
28+
tee ~/Desktop/blech_clust/github.log;
29+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
30+
then echo "ERROR detected by bash"; exit 1; fi
31+
EMG-Only:
32+
runs-on: self-hosted
33+
needs: Preamble
34+
steps:
35+
- name: Set up repo
36+
uses: actions/checkout@v4
37+
- name: Prefect EMG only test
38+
shell: bash
39+
working-directory: /home/exouser/Desktop/blech_clust
40+
run: python pipeline_testing/prefect_pipeline.py -e 2>&1 |
41+
tee ~/Desktop/blech_clust/github.log;
42+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
43+
then echo "ERROR detected by bash"; exit 1; fi
44+
Spike-EMG:
45+
runs-on: self-hosted
46+
needs: Preamble
47+
steps:
48+
- name: Set up repo
49+
uses: actions/checkout@v4
50+
- name: Prefect SPIKE then EMG test
51+
shell: bash
52+
working-directory: /home/exouser/Desktop/blech_clust
53+
run: python pipeline_testing/prefect_pipeline.py --spike-emg 2>&1 |
54+
tee ~/Desktop/blech_clust/github.log;
55+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
56+
then echo "ERROR detected by bash"; exit 1; fi
57+

‎pipeline_testing/prefect_pipeline.py

Lines changed: 123 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from glob import glob
99
import json
1010
import argparse
11+
import sys
1112

1213
############################################################
1314
parser = argparse.ArgumentParser(description='Run tests, default = Run all tests')
@@ -27,8 +28,18 @@
2728
help = 'Run all tests')
2829
parser.add_argument('--spike-emg', action = 'store_true',
2930
help = 'Run spike + emg in single test')
31+
parser.add_argument('--raise-exception', action = 'store_true',
32+
help = 'Raise error if subprocess fails')
3033
args = parser.parse_args()
3134

35+
print(args.raise_exception)
36+
break_bool = args.raise_exception
37+
38+
if break_bool:
39+
print('====================')
40+
print('Raising error if subprocess fails')
41+
print('====================')
42+
3243
def raise_error_if_error(process, stderr, stdout):
3344
# Print current data_type
3445
current_data_type_path = os.path.join(data_dir, 'current_data_type.txt')
@@ -408,96 +419,148 @@ def run_emg_freq_test(use_BSA = 1):
408419
emg_freq_post_process(data_dir)
409420
emg_freq_plot(data_dir)
410421

422+
##############################
411423
@flow(log_prints=True)
412424
def run_EMG_QDA_test():
413-
for data_type in ['emg', 'emg_spike']:
414-
print(f'Running QDA test with data type : {data_type}')
415-
try:
425+
if break_bool:
426+
for data_type in ['emg', 'emg_spike']:
427+
print(f'Running QDA test with data type : {data_type}')
416428
prep_data_flow(data_type = data_type)
417-
except:
418-
print('Failed to prep data')
419-
try:
420429
run_emg_main_test()
421430
os.chdir(os.path.join(blech_clust_dir, 'emg', 'gape_QDA_classifier'))
422431
run_gapes_Li(data_dir)
423-
except:
424-
print('Failed to run QDA test')
432+
else:
433+
for data_type in ['emg', 'emg_spike']:
434+
print(f'Running QDA test with data type : {data_type}')
435+
try:
436+
prep_data_flow(data_type = data_type)
437+
except:
438+
print('Failed to prep data')
439+
try:
440+
run_emg_main_test()
441+
os.chdir(os.path.join(blech_clust_dir, 'emg', 'gape_QDA_classifier'))
442+
run_gapes_Li(data_dir)
443+
except:
444+
print('Failed to run QDA test')
425445

426446
@flow(log_prints=True)
427447
def spike_only_test():
428-
for data_type in ['spike', 'emg_spike']:
429-
print(f'Running spike test with data type : {data_type}')
430-
try:
448+
if break_bool:
449+
for data_type in ['spike', 'emg_spike']:
450+
print(f'Running spike test with data type : {data_type}')
431451
prep_data_flow(data_type = data_type)
432-
except:
433-
print('Failed to prep data')
434-
try:
435452
run_spike_test()
436-
except:
437-
print('Failed to run spike test')
453+
else:
454+
for data_type in ['spike', 'emg_spike']:
455+
print(f'Running spike test with data type : {data_type}')
456+
try:
457+
prep_data_flow(data_type = data_type)
458+
except:
459+
print('Failed to prep data')
460+
try:
461+
run_spike_test()
462+
except:
463+
print('Failed to run spike test')
438464

439465
@flow(log_prints=True)
440466
def bsa_only_test():
441-
for data_type in ['emg', 'emg_spike']:
442-
print(f'Running BSA test with data type : {data_type}')
443-
try:
467+
if break_bool:
468+
for data_type in ['emg', 'emg_spike']:
469+
print(f'Running BSA test with data type : {data_type}')
444470
prep_data_flow(data_type = data_type)
445-
except:
446-
print('Failed to prep data')
447-
try:
448471
run_emg_freq_test(use_BSA=1)
449-
except:
450-
print('Failed to run emg BSA test')
472+
else:
473+
for data_type in ['emg', 'emg_spike']:
474+
print(f'Running BSA test with data type : {data_type}')
475+
try:
476+
prep_data_flow(data_type = data_type)
477+
except:
478+
print('Failed to prep data')
479+
try:
480+
run_emg_freq_test(use_BSA=1)
481+
except:
482+
print('Failed to run emg BSA test')
451483

452484
@flow(log_prints=True)
453485
def stft_only_test():
454-
for data_type in ['emg', 'emg_spike']:
455-
print(f'Running STFT test with data type : {data_type}')
456-
try:
486+
if break_bool:
487+
for data_type in ['emg', 'emg_spike']:
488+
print(f'Running STFT test with data type : {data_type}')
457489
prep_data_flow(data_type = data_type)
458-
except:
459-
print('Failed to prep data')
460-
try:
461490
run_emg_freq_test(use_BSA=0)
462-
except:
463-
print('Failed to run emg STFT test')
491+
else:
492+
for data_type in ['emg', 'emg_spike']:
493+
print(f'Running STFT test with data type : {data_type}')
494+
try:
495+
prep_data_flow(data_type = data_type)
496+
except:
497+
print('Failed to prep data')
498+
try:
499+
run_emg_freq_test(use_BSA=0)
500+
except:
501+
print('Failed to run emg STFT test')
464502

465503
@flow(log_prints=True)
466504
def run_emg_freq_only():
467-
try:
468-
bsa_only_test()
469-
except:
470-
print('Failed to run BSA test')
471-
try:
472-
stft_only_test()
473-
except:
474-
print('Failed to run STFT test')
505+
if break_bool:
506+
for data_type in ['emg', 'emg_spike']:
507+
print(f'Running EMG freq test with data type : {data_type}')
508+
prep_data_flow(data_type = data_type)
509+
run_emg_freq_test()
510+
else:
511+
try:
512+
bsa_only_test()
513+
except:
514+
print('Failed to run BSA test')
515+
try:
516+
stft_only_test()
517+
except:
518+
print('Failed to run STFT test')
475519

476520
@flow(log_prints=True)
477521
def emg_only_test():
478-
try:
479-
run_emg_freq_only()
480-
except:
481-
print('Failed to run emg freq test')
482-
try:
483-
run_EMG_QDA_test()
484-
except:
485-
print('Failed to run EMG QDA test')
522+
if break_bool:
523+
for data_type in ['emg', 'emg_spike']:
524+
print(f'Running EMG test with data type : {data_type}')
525+
prep_data_flow(data_type = data_type)
526+
run_emg_main_test()
527+
else:
528+
for data_type in ['emg', 'emg_spike']:
529+
print(f'Running EMG test with data type : {data_type}')
530+
try:
531+
prep_data_flow(data_type = data_type)
532+
except:
533+
print('Failed to prep data')
534+
try:
535+
run_emg_main_test()
536+
run_emg_freq_only()
537+
run_EMG_QDA_test()
538+
except:
539+
print('Failed to run emg test')
486540

487541
@flow(log_prints=True)
488542
def full_test():
489-
try:
490-
spike_only_test()
491-
except:
492-
print('Failed to run spike test')
493-
try:
494-
emg_only_test()
495-
except:
496-
print('Failed to run emg test')
497-
try:
498-
spike_emg_test()
499-
except:
500-
print('Failed to run spike + emg test')
543+
if break_bool:
544+
for data_type in ['emg', 'emg_spike']:
545+
print(f'Running full test with data type : {data_type}')
546+
prep_data_flow(data_type = data_type)
547+
run_spike_test()
548+
run_emg_main_test()
549+
else:
550+
for data_type in ['emg', 'emg_spike']:
551+
print(f'Running full test with data type : {data_type}')
552+
try:
553+
prep_data_flow(data_type = data_type)
554+
except:
555+
print('Failed to prep data')
556+
try:
557+
run_spike_test()
558+
except:
559+
print('Failed to run spike test')
560+
try:
561+
run_emg_main_test()
562+
except:
563+
print('Failed to run emg test')
501564

502565
############################################################
503566
## Run Flows

0 commit comments

Comments
 (0)