diff --git a/src/rosdep2/cache_tools.py b/src/rosdep2/cache_tools.py index 2930e88cd..44ede7d11 100644 --- a/src/rosdep2/cache_tools.py +++ b/src/rosdep2/cache_tools.py @@ -37,6 +37,7 @@ import pickle PICKLE_CACHE_EXT = '.pickle' +CACHE_PATH_ENV = 'ROSDEP_CACHE_PATH' def compute_filename_hash(key_filenames): diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 59ca82e65..7deaa34ca 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -63,7 +63,7 @@ from .installers import normalize_uninstalled_to_list from .installers import RosdepInstaller from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages -from .meta import MetaDatabase +from .meta import MetaDatabase, get_meta_cache_dir from .rospkg_loader import DEFAULT_VIEW_KEY from .sources_list import update_sources_list, get_sources_cache_dir,\ download_default_sources_list, SourcesListLoader, CACHE_INDEX,\ @@ -270,11 +270,12 @@ def setup_proxy_opener(): install_opener(opener) -def setup_environment_variables(ros_distro): +def setup_environment_variables(ros_distro, meta_cache_dir=None): """ Set environment variables needed to find ROS packages and evaluate conditional dependencies. :param ros_distro: The requested ROS distro passed on the CLI, or None + :param meta_cache_dir: Path to the cache directory of meta information """ if ros_distro is not None: if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != ros_distro: @@ -288,7 +289,7 @@ def setup_environment_variables(ros_distro): if 'ROS_PYTHON_VERSION' not in os.environ and 'ROS_DISTRO' in os.environ: # Set python version to version used by ROS distro - python_versions = MetaDatabase().get('ROS_PYTHON_VERSION', default=[]) + python_versions = MetaDatabase(meta_cache_dir).get('ROS_PYTHON_VERSION', default=[]) if os.environ['ROS_DISTRO'] in python_versions: os.environ['ROS_PYTHON_VERSION'] = str(python_versions[os.environ['ROS_DISTRO']]) @@ -301,12 +302,15 @@ def setup_environment_variables(ros_distro): def _rosdep_main(args): # sources cache dir is our local database. default_sources_cache = get_sources_cache_dir() + default_meta_cache = get_meta_cache_dir() parser = OptionParser(usage=_usage, prog='rosdep') parser.add_option('--os', dest='os_override', default=None, metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid') parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache, metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache)) + parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache, + metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache)) parser.add_option('--verbose', '-v', dest='verbose', default=False, action='store_true', help='verbose display') parser.add_option('--version', dest='print_version', default=False, @@ -434,7 +438,7 @@ def _rosdep_main(args): if command not in ['init', 'update', 'fix-permissions']: check_for_sources_list_init(options.sources_cache_dir) # _package_args_handler uses `ROS_DISTRO`, so environment variables must be set before - setup_environment_variables(options.ros_distro) + setup_environment_variables(options.ros_distro, options.meta_cache_dir) elif command not in ['fix-permissions']: setup_proxy_opener() @@ -666,7 +670,8 @@ def update_error_handler(data_source, exc): error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, ros_distro=options.ros_distro, - quiet=options.quiet) + quiet=options.quiet, + meta_cache_dir=options.meta_cache_dir) if not options.quiet: print('updated cache in %s' % (sources_cache_dir)) except InvalidData as e: diff --git a/src/rosdep2/meta.py b/src/rosdep2/meta.py index b8cd9006b..3ff574229 100644 --- a/src/rosdep2/meta.py +++ b/src/rosdep2/meta.py @@ -45,6 +45,7 @@ from .cache_tools import compute_filename_hash from .cache_tools import write_cache_file from .cache_tools import PICKLE_CACHE_EXT +from .cache_tools import CACHE_PATH_ENV """ Rosdep needs to store data that isn't used to resolve rosdep keys, but needs to be cached during @@ -56,6 +57,8 @@ def get_meta_cache_dir(): """Return storage location for cached meta data.""" + if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]: + return os.path.join(os.environ[CACHE_PATH_ENV], META_CACHE_DIR) ros_home = rospkg.get_ros_home() return os.path.join(ros_home, 'rosdep', META_CACHE_DIR) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index 04e1b6383..e9833a31e 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -37,7 +37,7 @@ except ImportError: import pickle -from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file +from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file, CACHE_PATH_ENV from .core import InvalidData, DownloadFailure, CachePermissionError from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data from .meta import MetaDatabase @@ -109,6 +109,8 @@ def get_default_sources_list_file(): def get_sources_cache_dir(): + if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]: + return os.path.join(os.environ[CACHE_PATH_ENV], SOURCES_CACHE_DIR) ros_home = rospkg.get_ros_home() return os.path.join(ros_home, 'rosdep', SOURCES_CACHE_DIR) @@ -435,7 +437,8 @@ def _generate_key_from_urls(urls): def update_sources_list(sources_list_dir=None, sources_cache_dir=None, success_handler=None, error_handler=None, skip_eol_distros=False, ros_distro=None, - quiet=False): + quiet=False, + meta_cache_dir=None): """ Re-downloaded data from remote sources and store in cache. Also update the cache index based on current sources. @@ -449,6 +452,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, if a particular source fails. This hook is mainly for printing errors to console. :param skip_eol_distros: skip downloading sources for EOL distros + :param meta_cache_dir: override meta cache directory :returns: list of (`DataSource`, cache_file_path) pairs for cache files that were updated, ``[str]`` @@ -516,7 +520,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, sources.append(rds) # cache metadata that isn't a source list - MetaDatabase().set('ROS_PYTHON_VERSION', python_versions) + MetaDatabase(meta_cache_dir).set('ROS_PYTHON_VERSION', python_versions) # Create a combined index of *all* the sources. We do all the # sources regardless of failures because a cache from a previous diff --git a/test/test_rosdep_sources_list.py b/test/test_rosdep_sources_list.py index dff036a4d..960bf10b0 100644 --- a/test/test_rosdep_sources_list.py +++ b/test/test_rosdep_sources_list.py @@ -214,13 +214,15 @@ def test_update_sources_list(): tempdir = tempfile.mkdtemp() # use a subdirectory of test dir to make sure rosdep creates the necessary substructure tempdir = os.path.join(tempdir, 'newdir') + metadir = os.path.join(tempdir, 'meta') errors = [] def error_handler(loc, e): errors.append((loc, e)) retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=error_handler) + sources_cache_dir=tempdir, error_handler=error_handler, + meta_cache_dir=metadir) assert retval assert len(retval) == 2, retval # one of our sources is intentionally bad, this should be a softfail @@ -253,6 +255,7 @@ def error_handler(loc, e): def test_load_cached_sources_list(): from rosdep2.sources_list import load_cached_sources_list, update_sources_list tempdir = tempfile.mkdtemp() + metadir = tempfile.mkdtemp() # test behavior on empty cache assert [] == load_cached_sources_list(sources_cache_dir=tempdir) @@ -260,7 +263,8 @@ def test_load_cached_sources_list(): # pull in cache data sources_list_dir = get_test_dir() retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=None) + sources_cache_dir=tempdir, error_handler=None, + meta_cache_dir=metadir) assert retval # now test with cached data @@ -422,11 +426,13 @@ def test_SourcesListLoader_create_default(): from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher # create temp dir for holding sources cache tempdir = tempfile.mkdtemp() + metadir = tempfile.mkdtemp() # pull in cache data sources_list_dir = get_test_dir() retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=None) + sources_cache_dir=tempdir, error_handler=None, + meta_cache_dir=metadir) assert retval # now test with cached data