Fixed some bugs related to the update of catalog with new events

mpi/template_manager.py

@@ -20,11 +20,11 @@ class TemplateManager(SequentialLoader):
             start, shift, mag = t
             if abs(start - template[0]) <= 0.1: #tolerance 0.1 seconds
                 self.logger.debug("duplicated template found (%s)" % str(template) )
-                return False
+                return False, -1
         next_id = max(self.map) + 1 if self.map else 0
         self.map[next_id] = template
         self.add(next_id)
-        return True
+        return True, next_id
 
     def get_template(self, id):
         return self.map[id]

mpi/template_matching.py

@@ -264,13 +264,13 @@ class TemplateMatching(object):
               not self.chunk_mngr.done()    or \
               not self.scanner_mngr.done():
 
+            # Reclaim any slave who completed his task and check results
+            self.__reclaim_slaves()
+
             # Possibly add a new template from UNDETECTED catalog events
             if self.first_scan:
                 self.__detect_new_templates()
 
-            # Reclaim any slave who completed his task and check results
-            self.__reclaim_slaves()
-
             # Load new templates (if any new one to load)
             self.__load_templates()
 
@@ -328,7 +328,6 @@ class TemplateMatching(object):
                 self.failed_templates.append(template)
             else:
                 self.template_mngr.set_loaded(template)
-                self.scanner_mngr.add_template_to_process(template)
                 self.logger.debug('(done) Loading template %d' % (template) )
 
         # reclaim readers and mark new chunks ready to be processed
@@ -341,7 +340,6 @@ class TemplateMatching(object):
                 self.logger.warning('(Error) Reading chunk %d' % (chunk) )
             else:
                 self.chunk_mngr.set_loaded(chunk)
-                self.scanner_mngr.add_chunk_to_process(chunk)
                 self.logger.debug('(done) Reading chunk %d' % (chunk) )
 
         # reclaim processors and mark chunk/template pair as processed
@@ -371,7 +369,6 @@ class TemplateMatching(object):
             # and it is not part of the the catalog, in this latter case
             # it will be ready after it has been checked for new templates
             if ( self.scanner_mngr.done([oldest_chunk]) and
-                 self.template_mngr.done()              and
                  oldest_chunk not in self.catalog_chunks ):
 
                 self.chunk_mngr.remove(oldest_chunk)
@@ -395,13 +392,9 @@ class TemplateMatching(object):
 
             #
             # check the next chunk from the catalog has been xcorred with all
-            # available templates, i.e. the chunk has been loaded and processed
-            # with all loaded templates and finally all the known templates have
-            # been already loaded
+            # available templates
             #
-            if ( not self.chunk_mngr.done(chunk_id)     or
-                 not self.scanner_mngr.done([chunk_id]) or
-                 not self.template_mngr.done() ):
+            if not self.scanner_mngr.done([chunk_id]):
                 break
 
             #
@@ -422,7 +415,9 @@ class TemplateMatching(object):
             missed_events = tmcore.missing_events(highCCC, self.catalog, starttime, endtime)
             for missed in missed_events:
                 new_template = (missed[1], 0.0, missed[2])
-                if self.template_mngr.add_template(new_template):
+                added, template_id = self.template_mngr.add_template(new_template)
+                if added:
+                    self.scanner_mngr.add_template_to_process(template_id)
                     self.logger.info('New template found: %s  | %.1f | M%4.2f' % tuple(new_template) )
                     return
 
@@ -501,10 +496,13 @@ class TemplateMatching(object):
             r = self.m_processors.move_slave(to_master=self.m_readers)
             if r is not None: self.logger.debug('Taking reader %d  back from processors' % (r) )
 
+        #
         # load chunks
+        #
         for r in self.m_readers.get_avaliable():
 
-            # get first not fully processed chunk
+            # get first chunk that need processing (if we resume a previous run
+            # some of the chunks might have been already processed)
             while True:
                 chunk = self.chunk_mngr.get_next()
                 if chunk is None or not self.scanner_mngr.done([chunk]):
@@ -519,6 +517,7 @@ class TemplateMatching(object):
                 self.m_readers.move_slave(to_master=self.m_processors, slave=r)
                 break
 
+            # finally load the chunk
             start, end, length = self.chunks.get(chunk)
             self.logger.debug('Reading chunk %d (start %s end %s length %s)' % (chunk, start, end, length))
             self.m_readers.run(r, data=(chunk, start, length) )