Add --meta-cache-dir option and ROSDEP_CACHE_PATH env variable.

Signed-off-by: Martin Pecka <[email protected]>

src/rosdep2/cache_tools.py

@@ -37,6 +37,7 @@
     import pickle
 
 PICKLE_CACHE_EXT = '.pickle'
+CACHE_PATH_ENV = 'ROSDEP_CACHE_PATH'
 
 
 def compute_filename_hash(key_filenames):

src/rosdep2/main.py

@@ -63,7 +63,7 @@
 from .installers import normalize_uninstalled_to_list
 from .installers import RosdepInstaller
 from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages
-from .meta import MetaDatabase
+from .meta import MetaDatabase, get_meta_cache_dir
 from .rospkg_loader import DEFAULT_VIEW_KEY
 from .sources_list import update_sources_list, get_sources_cache_dir,\
     download_default_sources_list, SourcesListLoader, CACHE_INDEX,\
@@ -270,11 +270,12 @@ def setup_proxy_opener():
             install_opener(opener)
 
 
-def setup_environment_variables(ros_distro):
+def setup_environment_variables(ros_distro, meta_cache_dir=None):
     """
     Set environment variables needed to find ROS packages and evaluate conditional dependencies.
 
     :param ros_distro: The requested ROS distro passed on the CLI, or None
+    :param meta_cache_dir: Path to the cache directory of meta information
     """
     if ros_distro is not None:
         if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != ros_distro:
@@ -288,7 +289,7 @@ def setup_environment_variables(ros_distro):
 
     if 'ROS_PYTHON_VERSION' not in os.environ and 'ROS_DISTRO' in os.environ:
         # Set python version to version used by ROS distro
-        python_versions = MetaDatabase().get('ROS_PYTHON_VERSION', default=[])
+        python_versions = MetaDatabase(meta_cache_dir).get('ROS_PYTHON_VERSION', default=[])
         if os.environ['ROS_DISTRO'] in python_versions:
             os.environ['ROS_PYTHON_VERSION'] = str(python_versions[os.environ['ROS_DISTRO']])
 
@@ -301,12 +302,15 @@ def setup_environment_variables(ros_distro):
 def _rosdep_main(args):
     # sources cache dir is our local database.
     default_sources_cache = get_sources_cache_dir()
+    default_meta_cache = get_meta_cache_dir()
 
     parser = OptionParser(usage=_usage, prog='rosdep')
     parser.add_option('--os', dest='os_override', default=None,
                       metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid')
     parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache,
                       metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache))
+    parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache,
+                      metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache))
     parser.add_option('--verbose', '-v', dest='verbose', default=False,
                       action='store_true', help='verbose display')
     parser.add_option('--version', dest='print_version', default=False,
@@ -434,7 +438,7 @@ def _rosdep_main(args):
     if command not in ['init', 'update', 'fix-permissions']:
         check_for_sources_list_init(options.sources_cache_dir)
         # _package_args_handler uses `ROS_DISTRO`, so environment variables must be set before
-        setup_environment_variables(options.ros_distro)
+        setup_environment_variables(options.ros_distro, options.meta_cache_dir)
     elif command not in ['fix-permissions']:
         setup_proxy_opener()
 
@@ -666,7 +670,8 @@ def update_error_handler(data_source, exc):
                             error_handler=update_error_handler,
                             skip_eol_distros=not options.include_eol_distros,
                             ros_distro=options.ros_distro,
-                            quiet=options.quiet)
+                            quiet=options.quiet,
+                            meta_cache_dir=options.meta_cache_dir)
         if not options.quiet:
             print('updated cache in %s' % (sources_cache_dir))
     except InvalidData as e:

src/rosdep2/meta.py

@@ -45,6 +45,7 @@
 from .cache_tools import compute_filename_hash
 from .cache_tools import write_cache_file
 from .cache_tools import PICKLE_CACHE_EXT
+from .cache_tools import CACHE_PATH_ENV
 
 """
 Rosdep needs to store data that isn't used to resolve rosdep keys, but needs to be cached during
@@ -56,6 +57,8 @@
 
 def get_meta_cache_dir():
     """Return storage location for cached meta data."""
+    if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]:
+        return os.path.join(os.environ[CACHE_PATH_ENV], META_CACHE_DIR)
     ros_home = rospkg.get_ros_home()
     return os.path.join(ros_home, 'rosdep', META_CACHE_DIR)
 

src/rosdep2/sources_list.py

@@ -37,7 +37,7 @@
 except ImportError:
     import pickle
 
-from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file
+from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file, CACHE_PATH_ENV
 from .core import InvalidData, DownloadFailure, CachePermissionError
 from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data
 from .meta import MetaDatabase
@@ -109,6 +109,8 @@ def get_default_sources_list_file():
 
 
 def get_sources_cache_dir():
+    if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]:
+        return os.path.join(os.environ[CACHE_PATH_ENV], SOURCES_CACHE_DIR)
     ros_home = rospkg.get_ros_home()
     return os.path.join(ros_home, 'rosdep', SOURCES_CACHE_DIR)
 
@@ -435,7 +437,8 @@ def _generate_key_from_urls(urls):
 def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
                         success_handler=None, error_handler=None,
                         skip_eol_distros=False, ros_distro=None,
-                        quiet=False):
+                        quiet=False,
+                        meta_cache_dir=None):
     """
     Re-downloaded data from remote sources and store in cache.  Also
     update the cache index based on current sources.
@@ -449,6 +452,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         if a particular source fails.  This hook is mainly for
         printing errors to console.
     :param skip_eol_distros: skip downloading sources for EOL distros
+    :param meta_cache_dir: override meta cache directory
 
     :returns: list of (`DataSource`, cache_file_path) pairs for cache
         files that were updated, ``[str]``
@@ -516,7 +520,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         sources.append(rds)
 
     # cache metadata that isn't a source list
-    MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)
+    MetaDatabase(meta_cache_dir).set('ROS_PYTHON_VERSION', python_versions)
 
     # Create a combined index of *all* the sources.  We do all the
     # sources regardless of failures because a cache from a previous

test/test_rosdep_sources_list.py

@@ -214,13 +214,15 @@ def test_update_sources_list():
     tempdir = tempfile.mkdtemp()
     # use a subdirectory of test dir to make sure rosdep creates the necessary substructure
     tempdir = os.path.join(tempdir, 'newdir')
+    metadir = os.path.join(tempdir, 'meta')
 
     errors = []
 
     def error_handler(loc, e):
         errors.append((loc, e))
     retval = update_sources_list(sources_list_dir=sources_list_dir,
-                                 sources_cache_dir=tempdir, error_handler=error_handler)
+                                 sources_cache_dir=tempdir, error_handler=error_handler,
+                                 meta_cache_dir=metadir)
     assert retval
     assert len(retval) == 2, retval
     # one of our sources is intentionally bad, this should be a softfail
@@ -253,14 +255,16 @@ def error_handler(loc, e):
 def test_load_cached_sources_list():
     from rosdep2.sources_list import load_cached_sources_list, update_sources_list
     tempdir = tempfile.mkdtemp()
+    metadir = tempfile.mkdtemp()
 
     # test behavior on empty cache
     assert [] == load_cached_sources_list(sources_cache_dir=tempdir)
 
     # pull in cache data
     sources_list_dir = get_test_dir()
     retval = update_sources_list(sources_list_dir=sources_list_dir,
-                                 sources_cache_dir=tempdir, error_handler=None)
+                                 sources_cache_dir=tempdir, error_handler=None,
+                                 meta_cache_dir=metadir)
     assert retval
 
     # now test with cached data
@@ -422,11 +426,13 @@ def test_SourcesListLoader_create_default():
     from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher
     # create temp dir for holding sources cache
     tempdir = tempfile.mkdtemp()
+    metadir = tempfile.mkdtemp()
 
     # pull in cache data
     sources_list_dir = get_test_dir()
     retval = update_sources_list(sources_list_dir=sources_list_dir,
-                                 sources_cache_dir=tempdir, error_handler=None)
+                                 sources_cache_dir=tempdir, error_handler=None,
+                                 meta_cache_dir=metadir)
     assert retval
 
     # now test with cached data