reduce the mem of latent_to_gene.py

Author: Ganten-Hornby

src/GPS/find_latent_representation.py

@@ -177,7 +177,30 @@ def run_find_latent_representation(args:FindLatentRepresentationsConfig):
 
     else:
         args = parser.parse_args()
-    config=FindLatentRepresentationsConfig(**vars(args))
+    config=FindLatentRepresentationsConfig(**{'annotation': 'SubClass',
+ 'convergence_threshold': 0.0001,
+ 'epochs': 300,
+ 'feat_cell': 3000,
+ 'feat_hidden1': 256,
+ 'feat_hidden2': 128,
+ 'gcn_decay': 0.01,
+ 'gcn_hidden1': 64,
+ 'gcn_hidden2': 30,
+ 'gcn_lr': 0.001,
+ 'hierarchically': False,
+ 'input_hdf5_path': '/storage/yangjianLab/songliyang/SpatialData/Data/Brain/macaque/Cell/processed/h5ad/T862_macaque3.h5ad',
+ 'label_w': 1.0,
+ 'n_comps': 300,
+ 'n_neighbors': 11,
+ 'nheads': 3,
+ 'output_hdf5_path': 'T862_macaque3/find_latent_representations/T862_macaque3_add_latent.h5ad',
+ 'p_drop': 0.1,
+ 'rec_w': 1.0,
+ 'sample_name': 'T862_macaque3',
+ 'type': 'SCT',
+ 'var': False,
+ 'weighted_adj': False})
+    # config=FindLatentRepresentationsConfig(**vars(args))
     start_time = time.time()
     logger.info(f'Find latent representations for {config.sample_name}...')
     pprint.pprint(config)

src/GPS/latent_to_gene.py

@@ -115,11 +115,10 @@ def _compute_regional_mkscore(cell_tg, ):
     gene_ranks_region[gene_ranks_region <= 1] = 0
 
     # Simultaneously consider the ratio of expression fractions and ranks
-    gene_ranks_region = pd.DataFrame(gene_ranks_region * frac_region)
-    gene_ranks_region.columns = [cell_tg]
+    gene_ranks_region = (gene_ranks_region * frac_region).values
 
     mkscore = np.exp(gene_ranks_region ** 2) - 1
-    return mkscore
+    return mkscore.astype(np.float16, copy=False)
 
 
 def run_latent_to_gene(config: LatentToGeneConfig):
@@ -193,8 +192,7 @@ def run_latent_to_gene(config: LatentToGeneConfig):
                             desc="Finding markers (Rank-based approach) | cells")
     ]
     # Normalize the marker scores
-    mk_score = pd.concat(mk_score, axis=1)
-    mk_score.index = adata.var_names
+    mk_score=pd.DataFrame(np.vstack(mk_score).T, index=adata.var.index,columns=cell_list)
     # mk_score_normalized = mk_score.div(mk_score.sum())*1e+2
     # Remove the mitochondrial genes
     mt_genes = [gene for gene in mk_score.index if gene.startswith('MT-') or gene.startswith('mt-')]
@@ -248,8 +246,8 @@ def run_latent_to_gene(config: LatentToGeneConfig):
         )
     else:
         args = parser.parse_args()
+        config = LatentToGeneConfig(**vars(args))
     logger.info(f'Latent to gene for {args.sample_name}...')
-    config = LatentToGeneConfig(**vars(args))
     pprint.pprint(config)
     start_time = time.time()
     run_latent_to_gene(config)

test/GPS-snakemake-workflow-macaque.smk

@@ -23,7 +23,7 @@ for file in Path(root).glob('*.h5ad'):
 
 annotation = "SubClass"
 data_type = "SCT"
-# sample_names = ['T135_macaque1']
+# sample_names = ['T584_macaque2']
 num_processes = 20
 
 rule all:
@@ -38,7 +38,7 @@ rule test_run:
         [f'{sample_name}/generate_ldscore/{sample_name}_generate_ldscore_chr{chrom}.done' for sample_name in
          sample_names]
 
-localrules: find_latent_representations,latent_to_gene
+# localrules: find_latent_representations,latent_to_gene
 
 rule find_latent_representations:
     input:
@@ -67,8 +67,11 @@ rule find_latent_representations:
         convergence_threshold=1e-4,
         hierarchically=False
     threads:
-        1
+        2
     benchmark: '{sample_name}/find_latent_representations/{sample_name}_add_latent.h5ad.benchmark'
+    resources:
+        mem_mb_per_cpu=lambda wildcards, threads, attempt: 20_000 * np.log2(attempt + 1),
+        qos='huge'
     run:
         command = f"""
 GPS run_find_latent_representations \
@@ -119,10 +122,9 @@ rule latent_to_gene:
         annotation=annotation,
         type=data_type
     threads:
-        3
+        1
     resources:
-        mem_mb=80_000,
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 80_000 * np.log2(attempt + 1),
+        mem_mb_per_cpu=lambda wildcards, threads, attempt: 70_000 * np.log2(attempt + 1),
         qos='huge'
     benchmark: '{sample_name}/latent_to_gene/{sample_name}_gene_marker_score.feather.benchmark'
     run:
@@ -166,7 +168,7 @@ rule generate_ldscore:
     threads:
         3
     resources:
-        mem_mb_per_cpu=lambda wildcards, threads, attempt: 30_000 / threads * np.log2(attempt + 1),
+        mem_mb_per_cpu=lambda wildcards, threads, attempt: 45_000 / threads * np.log2(attempt + 1),
         qos='huge'
     shell:
         """
@@ -202,7 +204,7 @@ rule spatial_ldsc:
         sumstats_config_file='/storage/yangjianLab/chenwenhao/projects/202312_GPS/src/GPS/example/sumstats_config.yaml',
         all_chunk = 20
     threads:
-        3
+        2
     benchmark:
         '{sample_name}/spatial_ldsc/{sample_name}.spatial_ldsc.done.benchmark'
     resources: