Optimise the internal storage of the poll watcher to reduce memory re…

…quirements.

ChangeLog

@@ -5,17 +5,14 @@ ChangeLog
 
         release: stable micro release 1.2.1:
 
-        * configure.ac:
-          - Bump version to 1.2.1.
-          - Add AX_CXX_HAVE_HASH.
+        * configure.ac: bump version to 1.2.1.
         * fsw.7: updated.
         * m4/ax_cxx_namespaces.m4: added to check for C++ compiler namespace
           support.
-        * m4/ax_cxx_have_hash.m4: added to check for std::hash in the
-          <functional> header.
-        * poll_watcher.cpp: use hash<string> to calculate the hash of a path and
-          use it as a key in the internal watcher maps.
-        * poll_watcher.h: add hash<string> member.
+        * poll_watcher.cpp: optimise internal storage to reduce memory
+          requirements.
+        * poll_watcher.h: optimise internal storage to reduce memory
+          requirements.
         * README:
           - Updated.
           - Use markdown syntax.

NEWS

@@ -3,8 +3,7 @@ NEWS
 
 New in 1.2.1:
 
-  * Internally hash the cached paths in a poll watcher so that memory
-    requirements dramatically decrease.
+  * Optimise internal storage to reduce memory requirements.
 
 New in 1.2.0:
 

README

@@ -42,13 +42,12 @@ Usage recommendations are as follows:
     on either a per process or a system-wide basis.
   * If feasible, watch directories instead of watching files.
   * If none of the above applies, use the poll watcher.  The authors' experience
-    indicates that fsw requires approximately 600 MB or RAM memory to observe a
-    hierarchy of 500.000 files if std::hash is defined in <functional> otherwise
-    it requires approximately 1300 MB.  A common bottleneck of the poll watcher
-    is disk access, since stat()-ing a huge number of files may take a huge
-    amount of time.  In this case, the latency should be set to a sufficiently
-    large value in order to reduce the performance degradation that may result
-    from frequent disk access.
+    indicates that fsw requires approximately 480 MB or RAM memory to observe a
+    hierarchy of 500.000 files.  A common bottleneck of the poll watcher is disk
+    access, since stat()-ing a great number of files may take a huge amount of
+    time.  In this case, the latency should be set to a sufficiently large value
+    in order to reduce the performance degradation that may result from frequent
+    disk access.
 
 Getting fsw
 -----------

README.md

@@ -42,13 +42,12 @@ Usage recommendations are as follows:
     on either a per process or a system-wide basis.
   * If feasible, watch directories instead of watching files.
   * If none of the above applies, use the poll watcher.  The authors' experience
-    indicates that fsw requires approximately 600 MB or RAM memory to observe a
-    hierarchy of 500.000 files if std::hash is defined in <functional> otherwise
-    it requires approximately 1300 MB.  A common bottleneck of the poll watcher
-    is disk access, since stat()-ing a huge number of files may take a huge
-    amount of time.  In this case, the latency should be set to a sufficiently
-    large value in order to reduce the performance degradation that may result
-    from frequent disk access.
+    indicates that fsw requires approximately 480 MB or RAM memory to observe a
+    hierarchy of 500.000 files.  A common bottleneck of the poll watcher is disk
+    access, since stat()-ing a great number of files may take a huge amount of
+    time.  In this case, the latency should be set to a sufficiently large value
+    in order to reduce the performance degradation that may result from frequent
+    disk access.
 
 Getting fsw
 -----------

configure.ac

@@ -71,6 +71,4 @@ then
   )
 fi
 
-AX_CXX_HAVE_HASH
-
 AC_OUTPUT

poll_watcher.cpp

@@ -21,41 +21,30 @@ poll_watcher::~poll_watcher()
 
 void poll_watcher::initial_scan_callback(const string &path, struct stat &stat)
 {
-#if defined(HAVE_CXX_HASH)
-  size_t path_hash = str_hash(path);
-#else
-  string path_hash = path;
-#endif
-  previous_data->tracked_files[path] = path_hash;
-  previous_data->mtime[path_hash] = stat.st_mtimespec;
-  previous_data->ctime[path_hash] = stat.st_ctimespec;
+
+  watched_file_info wfi { stat.st_mtimespec.tv_sec, stat.st_ctimespec.tv_sec };
+  previous_data->tracked_files[path] = wfi;
 }
 
 void poll_watcher::intermediate_scan_callback(
     const string &path,
     struct stat &stat)
 {
 
-#if defined(HAVE_CXX_HASH)
-  size_t path_hash = str_hash(path);
-#else
-  string path_hash = path;
-#endif
-
-  new_data->tracked_files[path] = path_hash;
-  new_data->mtime[path_hash] = stat.st_mtimespec;
-  new_data->ctime[path_hash] = stat.st_ctimespec;
+  watched_file_info wfi { stat.st_mtimespec.tv_sec, stat.st_ctimespec.tv_sec };
+  new_data->tracked_files[path] = wfi;
 
   if (previous_data->tracked_files.count(path))
   {
+    watched_file_info pwfi = previous_data->tracked_files[path];
     vector<event_flag> flags;
 
-    if (stat.st_mtimespec.tv_sec > previous_data->mtime[path_hash].tv_sec)
+    if (stat.st_mtimespec.tv_sec > pwfi.mtime)
     {
       flags.push_back(event_flag::Updated);
     }
 
-    if (stat.st_ctimespec.tv_sec > previous_data->ctime[path_hash].tv_sec)
+    if (stat.st_ctimespec.tv_sec > pwfi.ctime)
     {
       flags.push_back(event_flag::AttributeModified);
     }
@@ -67,8 +56,6 @@ void poll_watcher::intermediate_scan_callback(
     }
 
     previous_data->tracked_files.erase(path);
-    previous_data->mtime.erase(path_hash);
-    previous_data->ctime.erase(path_hash);
   }
   else
   {

poll_watcher.h

@@ -13,17 +13,14 @@ typedef void (poll_watcher::*poll_watcher_scan_callback)(
     const string &path,
     struct stat &stat);
 
+typedef struct watched_file_info {
+  time_t mtime;
+  time_t ctime;
+} watched_file_info;
+
 typedef struct poll_watcher_data
 {
-#if defined(HAVE_CXX_HASH)
-  fsw_hash_map<string, size_t> tracked_files;
-  fsw_hash_map<size_t, struct timespec> mtime;
-  fsw_hash_map<size_t, struct timespec> ctime;
-#else
-  fsw_hash_map<string, string> tracked_files;
-  fsw_hash_map<string, struct timespec> mtime;
-  fsw_hash_map<string, struct timespec> ctime;
-#endif
+  fsw_hash_map<string, watched_file_info> tracked_files;
 } poll_watcher_data;
 
 class poll_watcher: public watcher
@@ -55,9 +52,6 @@ class poll_watcher: public watcher
 
   poll_watcher_data *previous_data;
   poll_watcher_data *new_data;
-#if defined(HAVE_CXX_HASH)
-  hash<string> str_hash;
-#endif
 
   vector<event> events;
   time_t curr_time;