give up on automating chunk length

for now

Author: LukeHearon

buzzcode/analysis.py

@@ -47,25 +47,6 @@ def translate_results(results, classes, digits=1):
 suffix_partial = '_buzzchunk.csv'
 
 
-def solve_memory(memory_allot, cpus, framehop_prop):
-    """ given memory allotment, number of processes, and framelength, solve for number of streamer processes, depth of buffer, and chunklength """
-    memory_remaining = memory_allot
-    memory_remaining = memory_remaining - (0.350*cpus)  # memory (in GB) required for single tensorflow process
-
-    memorydensity_audio = 2.4 / 3600  # gigabytes of memory per second of decoded audio (estimate)  # TODO: re-test with memory profile; just give best guess at peak memory usage
-    audio_time_free = memory_remaining/memorydensity_audio
-    frame_time_free = audio_time_free * framehop_prop
-
-    # this is total guesswork. TODO: test! Tune!
-    concurrent_streamers = (cpus/2).__ceil__()  # on SSD, ideal seems to be near cpus/2 (when running with GPU also!)
-    buffer_max = 3 * cpus
-    chunks_at_once = concurrent_streamers + buffer_max
-
-    chunklength = int(frame_time_free/chunks_at_once)
-
-    return concurrent_streamers, buffer_max, chunklength
-
-
 def melt_coverage(cover_df, framelength=None):
     """ where cover_df is a dataframe with start and end columns OR framelength is provided"""
     if 'end' not in cover_df.columns and framelength is None:

buzzcode/analyze_audio.py

@@ -8,7 +8,7 @@
 setthreads(1)
 
 from buzzcode.embedders import load_embedder_model, load_embedder_config
-from buzzcode.analysis import load_model, translate_results, suffix_result, suffix_partial, solve_memory, melt_coverage, \
+from buzzcode.analysis import load_model, translate_results, suffix_result, suffix_partial, melt_coverage, \
     get_gaps, smooth_gaps, gaps_to_chunklist, stitch_partial
 from buzzcode.audio import stream_to_queue, get_duration
 import pandas as pd
@@ -22,7 +22,7 @@
 from datetime import datetime
 
 
-def analyze_batch(modelname, cpus, memory_allot, gpu=False, vram=None, embeddername='yamnet', framehop_prop=1,
+def analyze_batch(modelname, chunklength=2000, cpus=2, gpu=False, embeddername='yamnet', framehop_prop=1,
                   dir_audio=dir_audio_in, verbosity=1):
     timer_total = Timer()
 
@@ -82,12 +82,8 @@ def worker_logger():
     framelength_str = re.sub('^.*\\.', '', framelength_str)
     framelength_digits = len(framelength_str)
 
-
-    concurrent_streamers, buffer_max, chunklength = solve_memory(
-        memory_allot=memory_allot,
-        cpus=cpus,
-        framehop_prop=framehop_prop
-    )
+    concurrent_streamers = 2
+    buffer_max = 2
 
     if chunklength < framelength:
         raise ValueError(f"insufficient memory allotment")
@@ -316,7 +312,7 @@ def analyze_assignment(assignment):
         f"input directory: {dir_audio}\n"
         f"model: {modelname}\n"
         f"CPU count: {cpus}\n"
-        f"memory allotment {memory_allot}\n",
+        f"GPU count: {gpu}\n",
         0)
 
     proc_writer = multiprocessing.Process(target=worker_writer, name='writer_proc', args=[])
@@ -370,4 +366,4 @@ def analyze_assignment(assignment):
 
 
 if __name__ == "__main__":
-    analyze_batch(modelname='model_general', gpu=False, vram=1, cpus=4, memory_allot=10, verbosity=2)
+    analyze_batch(modelname='model_general', dir_audio='/media/server storage/experiments', gpu=True,cpus=0, verbosity=2)

buzzdetect.py

@@ -25,7 +25,6 @@ def str2bool(v):
 parser_analyze.add_argument('--modelname', help='the name of the directory holding the model data', required=True,
                             type=str)
 parser_analyze.add_argument('--cpus', required=True, type=int)
-parser_analyze.add_argument('--memory', required=True, type=float)
 parser_analyze.add_argument('--classes', required=False, type=str)  # give as...comma-separated list?
 parser_analyze.add_argument('--dir_audio', required=False, default="./audio_in", type=str)
 parser_analyze.add_argument('--dir_out', required=False, default=None, type=str)
@@ -65,7 +64,6 @@ def str2bool(v):
     analyze_batch(
         modelname=args.modelname,
         cpus=args.cpus,
-        memory_allot=args.memory,
         dir_audio=args.dir_audio,
         verbosity=args.verbosity,
     )