Change: progress calculation (#952)

Ospd receives the excluded hosts count from openvas. In the end, is openvas which knows how many hosts were excluded, after removing duplicated, or those which doesn't satisfy a preference like reverse_lookup_only.

However, excluded hosts list (and count) sent to openvas still includes de finished host if a scan is a resumed task
. Therefore, still progress calculation adds the finished hosts to total host reported by openvas.

For testing, run scans, stop them and resume it. Different targets should be tested, like a network with the reverse_lookup_only preference, excluded hosts, excluded hosts which do not belong to the target list, etc.

ospd/ospd.py

@@ -817,12 +817,13 @@ def _get_scan_progress_raw(self, scan_id: str) -> Dict:
         current_progress['count_dead'] = self.scan_collection.get_count_dead(
             scan_id
         )
-        current_progress['count_excluded'] = (
+        current_progress['count_total_excluded'] = (
             self.scan_collection.get_simplified_exclude_host_count(scan_id)
         )
+
         current_progress['count_total'] = self.scan_collection.get_count_total(
             scan_id
-        )
+        ) + self.scan_collection.get_finished_hosts_count(scan_id)
 
         logging.debug(
             "%s: Current progress: \n%s",
@@ -1256,6 +1257,15 @@ def set_scan_total_hosts(self, scan_id: str, count_total: int) -> None:
         the total count of host to be scanned."""
         self.scan_collection.update_count_total(scan_id, count_total)
 
+    def set_scan_total_excluded_hosts(
+        self, scan_id: str, excluded_hosts: int
+    ) -> None:
+        """Sets a scan's total excluded hosts. Allow the scanner to update
+        the total excluded count of hosts from the host to be scanned."""
+        self.scan_collection.update_count_total_excluded(
+            scan_id, excluded_hosts
+        )
+
     def clean_forgotten_scans(self) -> None:
         """Check for old stopped or finished scans which have not been
         deleted and delete them if the are older than the set value."""

ospd/scan.py

@@ -279,6 +279,7 @@ def unpickle_scan_info(self, scan_id: str) -> None:
         scan_info['count_alive'] = 0
         scan_info['count_dead'] = 0
         scan_info['count_total'] = None
+        scan_info['count_total_excluded'] = 0
         scan_info['excluded_simplified'] = None
         scan_info['target'] = unpickled_scan_info.pop('target')
         scan_info['vts'] = unpickled_scan_info.pop('vts')
@@ -383,6 +384,19 @@ def update_count_total(self, scan_id: str, count_total: int) -> int:
 
         self.scans_table[scan_id]['count_total'] = count_total
 
+    def update_count_total_excluded(
+        self, scan_id: str, count_excluded: int
+    ) -> int:
+        """Sets a scan's total hosts."""
+
+        self.scans_table[scan_id]['count_total_excluded'] = count_excluded
+
+    def get_count_total_excluded(self, scan_id: str) -> int:
+        """Get a scan's total host count."""
+
+        count_excluded = self.scans_table[scan_id]['count_total_excluded']
+        return count_excluded
+
     def get_count_total(self, scan_id: str) -> int:
         """Get a scan's total host count."""
 
@@ -479,15 +493,15 @@ def calculate_target_progress(self, scan_id: str) -> int:
         in the target."""
 
         total_hosts = self.get_count_total(scan_id)
-        exc_hosts = self.get_simplified_exclude_host_count(scan_id)
         count_alive = self.get_count_alive(scan_id)
         count_dead = self.get_count_dead(scan_id)
         host_progresses = self.get_current_target_progress(scan_id)
+        finished_hosts = self.get_finished_hosts_count(scan_id)
 
         try:
             t_prog = int(
                 (sum(host_progresses.values()) + 100 * count_alive)
-                / (total_hosts - exc_hosts - count_dead)
+                / (total_hosts + finished_hosts - count_dead)
             )
         except ZeroDivisionError:
             # Consider the case in which all hosts are dead or excluded
@@ -546,6 +560,14 @@ def get_finished_hosts(self, scan_id: str) -> str:
         """Get the finished host list sent by the client for a given target."""
         return self.scans_table[scan_id]['target'].get('finished_hosts')
 
+    def get_finished_hosts_count(self, scan_id: str) -> int:
+        """Get the finished host list sent by the client for a given target."""
+        fin_hosts = target_str_to_list(self.get_finished_hosts(scan_id))
+        finish_count = 0
+        if fin_hosts:
+            finish_count = len(fin_hosts)
+        return finish_count
+
     def get_credentials(self, scan_id: str) -> Dict[str, Dict[str, str]]:
         """Get a scan's credential list. It return dictionary with
         the corresponding credential for a given target.

ospd_openvas/daemon.py

@@ -860,6 +860,7 @@ def report_results(self, results: list, scan_id: str) -> bool:
                 or res["result_type"] == "HOST_END"
             )
             host_count = res["result_type"] == "HOSTS_COUNT"
+            host_excluded = res["result_type"] == "HOSTS_EXCLUDED"
             vt_aux = None
 
             # URI is optional and containing must be checked
@@ -870,6 +871,7 @@ def report_results(self, results: list, scan_id: str) -> bool:
                 and not host_deny
                 and not start_end_msg
                 and not host_count
+                and not host_excluded
             ):
                 if not roid and res["result_type"] != 'ERRMSG':
                     logger.warning('Missing VT oid for a result')
@@ -962,6 +964,19 @@ def report_results(self, results: list, scan_id: str) -> bool:
                 except TypeError:
                     logger.debug('Error processing total host count')
 
+            # To update total excluded hosts
+            if res["result_type"] == 'HOSTS_EXCLUDED':
+                try:
+                    total_excluded = int(res["value"])
+                    logger.debug(
+                        '%s: Set total excluded counted by OpenVAS: %d',
+                        scan_id,
+                        total_excluded,
+                    )
+                    self.set_scan_total_excluded_hosts(scan_id, total_excluded)
+                except TypeError:
+                    logger.debug('Error processing total excluded hosts')
+
         # Insert result batch into the scan collection table.
         if len(res_list):
             self.scan_collection.add_result_list(scan_id, res_list)

tests/test_scan_and_result.py

@@ -1370,7 +1370,7 @@ def test_get_scan_progress_xml(self):
         current_hosts = progress.findall('host')
         self.assertEqual(len(current_hosts), 2)
 
-        count_excluded = progress.findtext('count_excluded')
+        count_excluded = progress.findtext('count_total_excluded')
         self.assertEqual(count_excluded, '0')
 
     def test_set_get_vts_version(self):