else:
assert False
- assert not os.path.exists(dataset_fn), 'The data file already exists!'
- assert not os.path.exists(dataset_keys_fn), 'The data keys file already exists!'
+ #assert not os.path.exists(dataset_fn), 'The data file already exists!'
+ #assert not os.path.exists(dataset_keys_fn), 'The data keys file already exists!'
# saving new dataset and single keys as well
cPickle.dump(dataset,open(dataset_fn,'w+'),protocol=2)
except:
print ('Error: There was a problem generating the subdirectory: %s' % dir_name)
+ assert checkSettings(settings),'Check your settings some entries were invalid!'
+
ddir = settings['dataset_dir']
settings['prediction_dataset_fn'] = jp(ddir,'prediction_data.pickle')
settings['prediction_dataset_keys_fn'] = jp(ddir,'prediction_data.keys.pickle')
settings['training_dataset_fn'] = jp(ddir,'training_data.pickle')
settings['training_dataset_keys_fn'] = jp(ddir,'training_data.keys.pickle')
-
try:
os.mkdir(settings['global_log_fn'])
except:
def parseSettings(filename):
settings = parseSettingsFile(filename)
settings = makeSettings(settings)
- assert checkSettings(settings),'Check your settings some entries were invalid!'
-
return settings
ClusterTask.__init__(self,settings)
- def CreateJobs():
+ def CreateJobs(self):
"""
"""
jp = os.path.join
- run = cPickle.load(open(settings['run_fn']))
+ run = cPickle.load(open(self.settings['run_fn']))
run['name'] = 'saved_run'
- param = settings['prediction_parameter_fn']
+ param = self.settings['prediction_parameter_fn']
- run['result_dir'] = settings['prediction_dir']
- dataset_fn = settings['prediction_dataset_fn']
- prediction_keys_fn = settings['prediction_dataset_keys_fn']
+ run['result_dir'] = self.settings['prediction_dir']
+ dataset_fn = self.settings['prediction_dataset_fn']
+ prediction_keys_fn = self.settings['prediction_dataset_keys_fn']
prediction_keys = cPickle.load(open(prediction_keys_fn))
chunks.append((c_name,prediction_keys[slice[0]:slice[1]]))
for c_name,current_chunk in chunks:
- current_job = KybJob(grid_predict.TaskStarter,[run,dataset_fn,current_chunk,param,c_name])
+ current_job = KybJob(gridtools.TaskStarter,[run,dataset_fn,current_chunk,param,c_name])
current_job.h_vmem = '20.0G'
#current_job.express = 'True'
for size in [len(elem) for name,elem in chunks]:
sum += size
- print 'Got %d job(s)' % len(functionJobs)
+ print 'Got %d job(s)' % len(self.functionJobs)
- def TaskStarter(run,dataset_fn,prediction_keys,param,set_name):
- """
-
- """
+def TaskStarter(run,dataset_fn,prediction_keys,param,set_name):
+ """
+
+ """
- qp = QPalma()
- qp.predict(run,dataset_fn,prediction_keys,param,set_name)
+ qp = QPalma()
+ qp.predict(run,dataset_fn,prediction_keys,param,set_name)
- return 'finished prediction of set %s.' % set_name
+ return 'finished prediction of set %s.' % set_name