Every time I run this loop, it only runs for 3-6 iterations before throwing me this error. I'm almost positive its due to a memory issue since hF.BHDMS_fit() is a really computationally expensive function. In order to restart the loop i have to clear my kernal in my jupyter notebook and run it again. I'm running this off a decently powerful server but I dont know how to get past this error, i even tried to use garbage collect. Any suggestions so i can run this loop without having to restart the kernal every 5 iterations?
Loop:
n = 100 # Number of neurons
start = 88 #Firing rate in Hz
finish = 95 #Firing rate in Hz
total_time = 1 # Total time in seconds
bin_size = 0.01 # Bin size in seconds
num_trials = 100 # Number of trials
stored_lambda = []
for r in range(start, finish):
spike_counts_matrix = simulate_trials_poisson_neurons(n, r, total_time, bin_size, num_trials)
possible_codewords = (hF.extract_unique_codewords(spike_counts_matrix, n_cores=4)).T
sampled_cw = possible_codewords[np.random.choice(possible_codewords.shape[0], size=299, replace=False), :]
dmat = hF.hamming_distance_matrix(sampled_cw)
fit = hF.BHMDS_fit(sampled_cw, 2, 100, dmat)
warnings.filterwarnings('ignore')
stored_lambda.append(fit[1]['lambda'])
#explicitly delete large objects
del spike_counts_matrix, possible_codewords, sampled_cw, dmat, fit
gc.collect()
error:
/tmp/ccl1ZBv9.s: Assembler messages:
/tmp/ccl1ZBv9.s: Fatal error: can't close /tmp/pystan_10oiqnom/tmp/pystan_10oiqnom/stanfit4anon_model_83093fdb500d02ab161db95a3a9ac536_299748516262806981.o: No space left on device
---------------------------------------------------------------------------
DistutilsExecError Traceback (most recent call last)
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/unixccompiler.py:185, in UnixCCompiler._compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts)
184 try:
--> 185 self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs)
186 except DistutilsExecError as msg:
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/ccompiler.py:1041, in CCompiler.spawn(self, cmd, **kwargs)
1040 def spawn(self, cmd, **kwargs):
-> 1041 spawn(cmd, dry_run=self.dry_run, **kwargs)
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/spawn.py:70, in spawn(cmd, search_path, verbose, dry_run, env)
69 cmd = cmd[0]
---> 70 raise DistutilsExecError(
71 "command {!r} failed with exit code {}".format(cmd, exitcode)
72 )
DistutilsExecError: command '/usr/bin/gcc' failed with exit code 1
During handling of the above exception, another exception occurred:
CompileError Traceback (most recent call last)
Cell In[12], line 15
13 sampled_cw = possible_codewords[np.random.choice(possible_codewords.shape[0], size=299, replace=False), :]
14 dmat = hF.hamming_distance_matrix(sampled_cw)
---> 15 fit = hF.BHMDS_fit(sampled_cw, 2, 100, dmat)
16 warnings.filterwarnings('ignore')
17 stored_lambda.append(fit[1]['lambda'])
File /mnt/cube/jugorman/BHDMS/code/helperFxnzBHMDS_JCG.py:276, in BHMDS_fit(pts, dim, n_batch, dmat)
273 dmat = 2.0 * dmat / np.max(dmat)
275 # embed points in a given dimension
--> 276 fit = HMDS.embed(dim, dmat)
277 return dmat, fit
278 else:
File /mnt/cube/jugorman/BHDMS/code/metric_HMDS.py:250, in embed(D, dij, initial_values, Niter)
227 """
228 Returns embedding of distance matrix and optimized parameter values
229 Inputs:
(...)
246 'crs' - radial coordinates of points after CM translation
247 """
249 #compile model
--> 250 hmds_m = stan.StanModel(model_code=HMDS_code, verbose=False)
252 N = len(dij)
253 dat = {'N':N, 'D':D, 'deltaij':dij}
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/pystan-2.22.1.0.dev0-py3.8-linux-x86_64.egg/pystan/model.py:396, in StanModel.__init__(self, file, charset, model_name, model_code, stanc_ret, include_paths, boost_lib, eigen_lib, verbose, obfuscate_model_name, extra_compile_args, allow_undefined, include_dirs, includes)
393 orig_stderr = pystan.misc._redirect_stderr()
395 try:
--> 396 build_extension.run()
397 finally:
398 if redirect_stderr:
399 # restore stderr
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py:345, in build_ext.run(self)
342 self.compiler.set_link_objects(self.link_objects)
344 # Now actually compile and link everything.
--> 345 self.build_extensions()
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py:467, in build_ext.build_extensions(self)
465 self._build_extensions_parallel()
466 else:
--> 467 self._build_extensions_serial()
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py:493, in build_ext._build_extensions_serial(self)
491 for ext in self.extensions:
492 with self._filter_build_errors(ext):
--> 493 self.build_extension(ext)
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/command/build_ext.py:548, in build_ext.build_extension(self, ext)
545 for undef in ext.undef_macros:
546 macros.append((undef,))
--> 548 objects = self.compiler.compile(
549 sources,
550 output_dir=self.build_temp,
551 macros=macros,
552 include_dirs=ext.include_dirs,
553 debug=self.debug,
554 extra_postargs=extra_args,
555 depends=ext.depends,
556 )
558 # XXX outdated variable, kept here in case third-part code
559 # needs it.
560 self._built_objects = objects[:]
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/ccompiler.py:600, in CCompiler.compile(self, sources, output_dir, macros, include_dirs, debug, extra_preargs, extra_postargs, depends)
598 except KeyError:
599 continue
--> 600 self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
602 # Return *all* object filenames, not just the ones we just built.
603 return objects
File ~/anaconda3/envs/BHMDS/lib/python3.8/site-packages/setuptools/_distutils/unixccompiler.py:187, in UnixCCompiler._compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts)
185 self.spawn(compiler_so + cc_args + [src, '-o', obj] + extra_postargs)
186 except DistutilsExecError as msg:
--> 187 raise CompileError(msg)
CompileError: command '/usr/bin/gcc' failed with exit code 1