Skip to content

Commit f6216a7

Browse files
abuzarmahmoodabuzarmahmood
committed
Merge branch 'master' of https://github.com/katzlabbrandeis/blech_clust into 203-misc-blech_post_process-tweaks-and-issues
2 parents 694ada5 + b7bf8c3 commit f6216a7

File tree

183 files changed

+13912
-28446
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

183 files changed

+13912
-28446
lines changed

‎.github/workflows/python_workflow_test.yml

Lines changed: 118 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,145 @@
1-
# Description: This is a test workflow for running a Python script on a self-hosted runner
2-
# Don't chain jobs together as any failure in one job will stop the workflow
1+
# GitHub Actions workflow for automated testing
2+
# See: https://docs.github.com/en/actions
3+
#
4+
# This workflow runs Python tests on a self-hosted runner
5+
# Jobs are independent to allow partial success/failure reporting
36

4-
name: Python test
5-
run-name: python_test
6-
on: [pull_request]
7+
name: Python test
8+
run-name: python_test
9+
on:
10+
pull_request:
11+
workflow_dispatch:
712
jobs:
813
Preamble:
914
runs-on: self-hosted
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.ref }}
17+
cancel-in-progress: true
1018
steps:
11-
- run: cd ~/Desktop/blech_clust
1219
- run: pwd
1320
- run: which python
1421
- run: conda info --envs
1522
- name: Set up repo
1623
uses: actions/checkout@v4
24+
- run: git status
1725
- run: echo "${{ github.ref }} | ${{ github.repository }} | ${{ github.event.pull_request.title }}"
18-
Spike-Only:
26+
- name: check labels
27+
run: echo "${{ github.event.pull_request.labels.*.name }}"
28+
- name: Copy files to working-directory
29+
run: cp -r ./* /home/exouser/Desktop/blech_clust
30+
- name: Check files in working-directory
31+
run: ls -R /home/exouser/Desktop/blech_clust
32+
- name: Setup params
33+
run: cp /home/exouser/Desktop/blech_clust/params/_templates/* /home/exouser/Desktop/blech_clust/params
34+
- name: Check params
35+
run: for f in $(find /home/exouser/Desktop/blech_clust/params/ -type f); do echo $f; cat $f; done
36+
Install:
1937
runs-on: self-hosted
2038
needs: Preamble
39+
if: ${{ contains(github.event.pull_request.labels.*.name, 'install') }}
40+
# Only run installation when PR has 'install' label
2141
steps:
22-
- name: Set up repo
23-
uses: actions/checkout@v4
24-
- name: Prefect SPIKE only test
42+
- name: Clean install
43+
working-directory: /home/exouser/Desktop/blech_clust
44+
run: |
45+
echo "Running installation for tag ${{ github.ref }}"
46+
make clean
47+
- name: Install dependencies
48+
working-directory: /home/exouser/Desktop/blech_clust
49+
run: |
50+
make base
51+
- name: Install EMG
52+
working-directory: /home/exouser/Desktop/blech_clust
53+
run: |
54+
make emg
55+
- name: Install neuRecommend
56+
working-directory: /home/exouser/Desktop/blech_clust
57+
run: |
58+
make neurec
59+
- name: Install BlechRNN
60+
working-directory: /home/exouser/Desktop/blech_clust
61+
run: |
62+
make blechrnn
63+
- name: Install Prefect
64+
working-directory: /home/exouser/Desktop/blech_clust
65+
run: |
66+
make prefect
67+
- name: Patch dependencies
68+
working-directory: /home/exouser/Desktop/blech_clust
69+
run: |
70+
make patch
71+
Spike-EMG-Install:
72+
runs-on: self-hosted
73+
needs: Install
74+
concurrency:
75+
group: ${{ github.workflow }}-${{ github.ref }}
76+
cancel-in-progress: true
77+
# Test full pipeline with both spike sorting and EMG analysis
78+
if: ${{ contains(github.event.pull_request.labels.*.name, 'install') }}
79+
steps:
80+
- name: Prefect SPIKE then EMG test
2581
shell: bash
2682
working-directory: /home/exouser/Desktop/blech_clust
27-
run: python pipeline_testing/prefect_pipeline.py -s 2>&1 |
28-
tee ~/Desktop/blech_clust/github.log;
29-
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
30-
then echo "ERROR detected by bash"; exit 1; fi
31-
EMG-Only:
83+
run: conda run -n blech_clust python pipeline_testing/prefect_pipeline.py --spike-emg 2>&1 |
84+
tee ~/Desktop/blech_clust/github.log;
85+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log; then
86+
echo "ERROR detected by bash";
87+
./pipeline_testing/extract_traceback.sh ~/Desktop/blech_clust/github.log;
88+
exit 1;
89+
fi
90+
EMG-Only-Install:
3291
runs-on: self-hosted
33-
needs: Preamble
92+
needs: Spike-EMG-Install
93+
concurrency:
94+
group: ${{ github.workflow }}-${{ github.ref }}
95+
cancel-in-progress: true
96+
# Test EMG analysis pipeline in isolation
3497
steps:
35-
- name: Set up repo
36-
uses: actions/checkout@v4
3798
- name: Prefect EMG only test
3899
shell: bash
39100
working-directory: /home/exouser/Desktop/blech_clust
40-
run: python pipeline_testing/prefect_pipeline.py -e 2>&1 |
41-
tee ~/Desktop/blech_clust/github.log;
42-
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
43-
then echo "ERROR detected by bash"; exit 1; fi
101+
run: conda run -n blech_clust python pipeline_testing/prefect_pipeline.py -e 2>&1 |
102+
tee ~/Desktop/blech_clust/github.log;
103+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log; then
104+
echo "ERROR detected by bash";
105+
./pipeline_testing/extract_traceback.sh ~/Desktop/blech_clust/github.log;
106+
exit 1;
107+
fi
44108
Spike-EMG:
45109
runs-on: self-hosted
46-
needs: Preamble
110+
needs: Preamble
111+
concurrency:
112+
group: ${{ github.workflow }}-${{ github.ref }}
113+
cancel-in-progress: true
114+
# Test full pipeline with both spike sorting and EMG analysis
115+
if: ${{ !contains(github.event.pull_request.labels.*.name, 'install') }}
47116
steps:
48-
- name: Set up repo
49-
uses: actions/checkout@v4
50117
- name: Prefect SPIKE then EMG test
51118
shell: bash
52119
working-directory: /home/exouser/Desktop/blech_clust
53-
run: python pipeline_testing/prefect_pipeline.py --spike-emg 2>&1 |
54-
tee ~/Desktop/blech_clust/github.log;
55-
if grep -q "ERROR" ~/Desktop/blech_clust/github.log;
56-
then echo "ERROR detected by bash"; exit 1; fi
57-
120+
run: conda run -n blech_clust python pipeline_testing/prefect_pipeline.py --spike-emg 2>&1 |
121+
tee ~/Desktop/blech_clust/github.log;
122+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log; then
123+
echo "ERROR detected by bash";
124+
./pipeline_testing/extract_traceback.sh ~/Desktop/blech_clust/github.log;
125+
exit 1;
126+
fi
127+
EMG-Only:
128+
runs-on: self-hosted
129+
needs: Spike-EMG
130+
concurrency:
131+
group: ${{ github.workflow }}-${{ github.ref }}
132+
cancel-in-progress: true
133+
# Test EMG analysis pipeline in isolation
134+
if: ${{ !contains(github.event.pull_request.labels.*.name, 'install') }}
135+
steps:
136+
- name: Prefect EMG only test
137+
shell: bash
138+
working-directory: /home/exouser/Desktop/blech_clust
139+
run: conda run -n blech_clust python pipeline_testing/prefect_pipeline.py -e 2>&1 |
140+
tee ~/Desktop/blech_clust/github.log;
141+
if grep -q "ERROR" ~/Desktop/blech_clust/github.log; then
142+
echo "ERROR detected by bash";
143+
./pipeline_testing/extract_traceback.sh ~/Desktop/blech_clust/github.log;
144+
exit 1;
145+
fi

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ requirements/.DS_Store
66
blech.dir
77
*.swp
88
*.Rhistory
9+
.aider*
10+
.env

‎.pre-commit-config.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# See https://pre-commit.com for more information
2+
# See https://pre-commit.com/hooks.html for more hooks
3+
repos:
4+
- repo: https://github.com/pre-commit/pre-commit-hooks
5+
rev: v5.0.0
6+
hooks:
7+
- id: check-json
8+
- id: trailing-whitespace
9+
- id: end-of-file-fixer
10+
- id: check-added-large-files
11+
- id: pretty-format-json
12+
args:
13+
- "--autofix"
14+
- "--indent=4"
15+
- "--no-sort-keys"
16+
17+
- repo: https://github.com/hhatto/autopep8
18+
rev: v2.3.1 # select the tag or revision you want, or run `pre-commit autoupdate`
19+
hooks:
20+
- id: autopep8
21+
args:
22+
- "--in-place"

‎Makefile

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
.PHONY: all base emg neurec blechrnn clean params patch
2+
3+
# Store sudo password
4+
# define get_sudo_password
5+
# $(eval SUDO_PASS := $(shell bash -c 'read -s -p "Enter sudo password: " pwd; echo $$pwd'))
6+
# @echo
7+
# endef
8+
9+
SCRIPT_DIR = $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))# This will return blech_clust base dir
10+
INSTALL_PATH = $(SCRIPT_DIR)/requirements/BaSAR_1.3.tar.gz
11+
INSTALL_STR = install.packages('$(INSTALL_PATH)', repos=NULL)
12+
13+
# Default target
14+
all: base emg neurec blechrnn prefect patch
15+
16+
# Create and setup base environment
17+
base: params
18+
$(call get_sudo_password)
19+
conda deactivate || true
20+
conda update -n base -c conda-forge conda -y
21+
conda clean --all -y
22+
conda create --name blech_clust python=3.8 -y
23+
conda run -n blech_clust conda install -c conda-forge -y --file requirements/conda_requirements_base.txt
24+
# bash requirements/install_gnu_parallel.sh "$(SUDO_PASS)"
25+
conda run -n blech_clust pip install --no-cache-dir -r requirements/pip_requirements_base.txt
26+
27+
# Install EMG (BSA) requirements
28+
emg:
29+
conda run -n blech_clust conda config --set channel_priority strict
30+
conda run -n blech_clust conda install -c conda-forge r-base=3.6 r-polynom r-orthopolynom -y
31+
# rpy2 has to be built against current R...caching messes with that
32+
conda run -n blech_clust pip install rpy2 --no-cache-dir
33+
# BaSAR is archived on CRAN, so we need to install it from a local file
34+
conda run -n blech_clust Rscript -e "${INSTALL_STR}"
35+
36+
# Install neuRecommend classifier
37+
neurec:
38+
@if [ ! -d ~/Desktop/neuRecommend ]; then \
39+
cd ~/Desktop && \
40+
git clone https://github.com/abuzarmahmood/neuRecommend.git; \
41+
else \
42+
echo "neuRecommend already exists"; \
43+
fi
44+
cd ~/Desktop && \
45+
conda run -n blech_clust pip install --no-cache-dir -r neuRecommend/requirements.txt
46+
47+
# Install BlechRNN (optional)
48+
blechrnn:
49+
@if [ ! -d ~/Desktop/blechRNN ]; then \
50+
cd ~/Desktop && \
51+
git clone https://github.com/abuzarmahmood/blechRNN.git; \
52+
else \
53+
echo "blechRNN already exists"; \
54+
fi
55+
cd ~/Desktop && \
56+
cd blechRNN && \
57+
conda run -n blech_clust pip install --no-cache-dir $$(cat requirements.txt | egrep "torch")
58+
59+
# Patch dependencies
60+
patch:
61+
conda run -n blech_clust bash requirements/patch_dependencies.sh
62+
63+
# Copy parameter templates
64+
# If more than 1 json file exists in params, don't copy templates and print warning
65+
# This is to prevent overwriting existing parameter files
66+
# If no json files exist, copy templates
67+
params:
68+
@if [ $$(ls params/*.json 2>/dev/null | wc -l) -gt 1 ]; then \
69+
echo "Warning: Params files detected in params dir. Not copying templates."; \
70+
elif [ $$(ls params/*.json 2>/dev/null | wc -l) -eq 1 ]; then \
71+
echo "Copying parameter templates to params directory"; \
72+
fi
73+
74+
# Install Prefect
75+
prefect:
76+
conda run -n blech_clust pip install --no-cache-dir -U prefect
77+
78+
# Clean up environments
79+
clean:
80+
conda env remove -n blech_clust -y

0 commit comments

Comments
 (0)