Merge: 9 -> main

Changelog.md

@@ -1,6 +1,31 @@
 ## Gazebo Fuel Tools 9.x
 
-### Gazebo Fuel Tools 9.0.0 (202X-XX-XX)
+### Gazebo Fuel Tools 9.0.0 (2023-09-29)
+
+1. Added script to update assets to gz
+    * [Pull request #377](https://github.com/gazebosim/gz-fuel-tools/pull/377)
+
+1. Documentation fixes
+    * [Pull request #371](https://github.com/gazebosim/gz-fuel-tools/pull/371)
+    * [Pull request #372](https://github.com/gazebosim/gz-fuel-tools/pull/372)
+    * [Pull request #375](https://github.com/gazebosim/gz-fuel-tools/pull/375)
+
+1. ign -> gz
+    * [Pull request #364](https://github.com/gazebosim/gz-fuel-tools/pull/364)
+    * [Pull request #375](https://github.com/gazebosim/gz-fuel-tools/pull/375)
+
+1. Remove GZ_FUEL_INITIAL_CONFIG_PATH macro
+    * [Pull request #363](https://github.com/gazebosim/gz-fuel-tools/pull/363)
+
+1. Infrastructure
+    * [Pull request #323](https://github.com/gazebosim/gz-fuel-tools/pull/323)
+    * [Pull request #374](https://github.com/gazebosim/gz-fuel-tools/pull/374)
+
+1. Update package dependencies for harmonic (gz-msgs10)
+    * [Pull request #326](https://github.com/gazebosim/gz-fuel-tools/pull/326)
+
+1. ⬆️  Bump main to 9.0.0~pre1
+    * [Pull request #283](https://github.com/gazebosim/gz-fuel-tools/pull/283)
 
 ## Gazebo Fuel Tools 8.x
 

README.md

@@ -21,7 +21,7 @@ interacting with Gazebo Fuel servers.
 
 # Building and installing
 
-See the [installation tutorial](https://gazebosim.org/api/fuel_tools/5.0/install.html).
+See the [installation tutorial](https://gazebosim.org/api/fuel_tools/9/install.html).
 
 Make sure `GZ_CONFIG_PATH` is set to the right install location so that `gz fuel` will work.
 Default is `/usr/local/share/gz`.
@@ -31,7 +31,7 @@ Default is `/usr/local/share/gz`.
 For a complete list of commands run `gz fuel -h` on the command line.
 
 **List all models**
-```
+```{.sh}
 $ gz fuel list -t model -r | head
 https://fuel.gazebosim.org/anonymous/test_model_595389531
 https://fuel.gazebosim.org/anonymous/test_model_122023392
@@ -46,7 +46,7 @@ https://fuel.gazebosim.org/anonymous/test_model_380348669
 ```
 
 **Download a model**
-```
+```{.sh}
 $ gz fuel download -u https://fuel.gazebosim.org/1.0/OpenRobotics/models/Ambulance -v 4
 Downloading model:
   Name: Ambulance
@@ -59,7 +59,7 @@ Download succeeded.
 ```
 
 **C++ Get List models**
-```
+```cpp
   // Create a client (uses https://fuel.gazebosim.org by default)
   gz::fuel_tools::ClientConfig conf;
   gz::fuel_tools::FuelClient client(conf);
@@ -80,22 +80,13 @@ Create a private token at
 [https://app.gazebosim.org/settings#access_tokens](https://app.gazebosim.org/settings#access_tokens). Store the generated token someplace safe.
 
 The private token can then used to upload a model:
-```
+```{.sh}
 gz fuel upload -m ~/path_to_model --header 'Private-token: <TOKEN>'
 ```
-## TODO
-
-See issues beginning with [Fuel backend] in the title. Here are two examples.
-
-**TODO: Find a model on disk**
-```
-$ gz fuel locate --name am1
-/home/developer/.ignition/fuel/fuel.gazebosim.org/alice/am1
-```
 
 ## Dependencies
 On ubuntu run
-```
+```{.sh}
 sudo apt install ruby-ffi libzip-dev libcurl-dev libjsoncpp-dev
 ```
 

src/ClientConfig.cc

@@ -44,13 +44,6 @@ class gz::fuel_tools::ClientConfigPrivate
                 homePath, ".gz", "fuel");
 
             this->servers.push_back(ServerConfig());
-            // Add in fuel.ignitionrobotics.org as another default server
-            // config.
-            ServerConfig ignServerConfig;
-            ignServerConfig.SetUrl(
-                common::URI("https://fuel.ignitionrobotics.org"));
-            ignServerConfig.SetVersion("1.0");
-            this->servers.push_back(ignServerConfig);
           }
 
   /// \brief Clear values.
@@ -60,7 +53,7 @@ class gz::fuel_tools::ClientConfigPrivate
             this->cacheLocation = "";
             this->configPath = "";
             this->userAgent =
-              "IgnitionFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL;
+              "GazeboFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL;
           }
 
   /// \brief A list of servers.
@@ -74,7 +67,7 @@ class gz::fuel_tools::ClientConfigPrivate
 
   /// \brief Name of the user agent.
   public: std::string userAgent =
-          "IgnitionFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL;
+          "GazeboFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL;
 };
 
 //////////////////////////////////////////////////

src/ClientConfig_TEST.cc

@@ -72,7 +72,7 @@ class ServerConfigTest: public ClientConfigTest {};
 TEST_F(ClientConfigTest, InitiallyDefaultServers)
 {
   ClientConfig config;
-  EXPECT_EQ(2u, config.Servers().size());
+  EXPECT_EQ(1u, config.Servers().size());
 }
 
 /////////////////////////////////////////////////
@@ -84,7 +84,7 @@ TEST_F(ClientConfigTest, ServersCanBeAdded)
   srv.SetUrl(common::URI("http://asdf"));
   config.AddServer(srv);
 
-  ASSERT_EQ(3u, config.Servers().size());
+  ASSERT_EQ(2u, config.Servers().size());
   EXPECT_EQ(std::string("http://asdf"), config.Servers().back().Url().Str());
 }
 
@@ -93,12 +93,9 @@ TEST_F(ClientConfigTest, ServersCanBeAdded)
 TEST_F(ClientConfigTest, CustomDefaultConfiguration)
 {
   ClientConfig config;
-  ASSERT_EQ(2u, config.Servers().size());
+  ASSERT_EQ(1u, config.Servers().size());
   EXPECT_EQ("https://fuel.gazebosim.org",
     config.Servers().front().Url().Str());
-  EXPECT_EQ("https://fuel.ignitionrobotics.org",
-    config.Servers()[1].Url().Str());
-
   std::string defaultCacheLocation = gz::common::joinPaths(
     homePath(), ".gz", "fuel");
   EXPECT_EQ(defaultCacheLocation, config.CacheLocation());
@@ -132,13 +129,9 @@ TEST_F(ClientConfigTest, CustomConfiguration)
 
   EXPECT_TRUE(config.LoadConfig(testPath));
 
-  ASSERT_EQ(4u, config.Servers().size());
+  ASSERT_EQ(3u, config.Servers().size());
   EXPECT_EQ("https://fuel.gazebosim.org",
     config.Servers().front().Url().Str());
-  EXPECT_EQ("https://fuel.ignitionrobotics.org",
-    config.Servers()[1].Url().Str());
-  EXPECT_EQ("https://api.gazebosim.org",
-    config.Servers()[2].Url().Str());
   EXPECT_EQ("https://myserver",
     config.Servers().back().Url().Str());
 
@@ -261,7 +254,7 @@ TEST_F(ClientConfigTest, EmptyCachePathConfiguration)
 TEST_F(ClientConfigTest, UserAgent)
 {
   ClientConfig config;
-  EXPECT_EQ("IgnitionFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL,
+  EXPECT_EQ("GazeboFuelTools-" GZ_FUEL_TOOLS_VERSION_FULL,
             config.UserAgent());
 
   config.SetUserAgent("my_user_agent");

src/FuelClient.cc

@@ -200,12 +200,6 @@ class gz::fuel_tools::FuelClientPrivate
   /// license information.
   public: void PopulateLicenses(const ServerConfig &_server);
 
-  /// \brief Checks the provided URI for fuel.gazebosim.org, and
-  /// prints a deprecation warning message if found.
-  /// \param[in] _uri URI to check
-  /// DEPRECATED/DEPRECATION: remove this function in Gazebo H.
-  public: void CheckForDeprecatedUri(const common::URI &_uri);
-
   /// \brief Get zip data from a REST response. This is used by world and
   /// model download.
   public: void ZipFromResponse(const RestResponse &_resp,
@@ -529,7 +523,7 @@ Result FuelClient::UploadModel(const std::string &_pathToModelDir,
            << "  1. Is the Server URL correct? Try entering it on a browser.\n"
            << "  2. Do the categories exist? If you are using the Fuel server,"
            << "     then you can get the complete list at"
-           << "     https://fuel.ignitionrobotics.org/1.0/categories.\n"
+           << "     https://fuel.gazebosim.org/1.0/categories.\n"
            << "  3. If the owner is specified, make sure you have correct\n"
            << "     permissions." << std::endl;
     return Result(ResultType::FETCH_ERROR);
@@ -564,7 +558,6 @@ void FuelClient::AddServerConfigParametersToHeaders(
 Result FuelClient::DeleteUrl(const gz::common::URI &_uri,
     const std::vector<std::string> &_headers)
 {
-  this->dataPtr->CheckForDeprecatedUri(_uri);
   gz::fuel_tools::Rest rest;
 
   RestResponse resp;
@@ -668,7 +661,6 @@ Result FuelClient::DownloadModel(const ModelIdentifier &_id,
           << std::endl << _id.Server().AsString() << std::endl;
     return Result(ResultType::FETCH_ERROR);
   }
-  this->dataPtr->CheckForDeprecatedUri(_id.Server().Url());
 
   // Route
   common::URIPath route;
@@ -841,8 +833,6 @@ Result FuelClient::DownloadWorld(WorldIdentifier &_id,
     return Result(ResultType::FETCH_ERROR);
   }
 
-  this->dataPtr->CheckForDeprecatedUri(_id.Server().Url());
-
   // Route
   common::URIPath route;
   route = route / _id.Owner() / "worlds" / _id.Name() / _id.VersionStr() /
@@ -1211,8 +1201,6 @@ bool FuelClient::ParseModelFileUrl(const common::URI &_modelFileUrl,
   if (!_modelFileUrl.Valid())
     return false;
 
-  this->dataPtr->CheckForDeprecatedUri(_modelFileUrl);
-
   auto urlStr = _modelFileUrl.Str();
 
   std::smatch match;
@@ -1940,19 +1928,6 @@ bool FuelClient::UpdateWorlds(const std::vector<std::string> &_headers)
   }
   return true;
 }
-//////////////////////////////////////////////////
-void FuelClientPrivate::CheckForDeprecatedUri(const common::URI &_uri)
-{
-  static std::string oldServer = "fuel.ignitionrobotics.org";
-  auto ignFuelPos = _uri.Str().find(oldServer);
-  if (ignFuelPos != std::string::npos)
-  {
-    std::string newUrl = _uri.Str();
-    newUrl.replace(ignFuelPos, oldServer.size(), "fuel.gazebosim.org");
-    gzwarn << "The " << oldServer << " URL is deprecrated. Pleasse change "
-      << _uri.Str() << " to " << newUrl << std::endl;
-  }
-}
 
 //////////////////////////////////////////////////
 void FuelClientPrivate::ZipFromResponse(const RestResponse &_resp,

src/FuelClient_TEST.cc

@@ -407,8 +407,7 @@ class FuelClientDownloadTest
 
 INSTANTIATE_TEST_SUITE_P(
     FuelClientTest, FuelClientDownloadTest,
-    ::testing::Values("fuel.gazebosim.org",
-                      "fuel.ignitionrobotics.org"));
+    ::testing::Values("fuel.gazebosim.org"));
 
 /////////////////////////////////////////////////
 // Protocol "https" not supported or disabled in libcurl for Windows

tools/gz-update-assets.py

@@ -0,0 +1,127 @@
+# Usage
+#     python3 gz-update-assets.py -o <collection_owner> -t <private_token>
+#
+# Description
+#     This script change usage of ignition to gz.
+#
+import sys,json,requests
+import getopt
+import fnmatch
+import os
+import re
+import subprocess
+import zipfile
+
+# Replace all occurrences of a string with another string inside all 
+# files with the provided extension. The extension is treated as
+# case-insentive.
+def find_and_replace(directory, find, replace, extension):
+  for root, dirs, files in os.walk(directory):
+    for file in files:
+      base, ext = os.path.splitext(file)
+      if ext.lower() == extension.lower():
+        filepath = os.path.join(root, file)
+        with open(filepath) as f:
+          contents = f.read()
+        contents = re.sub(find, replace, contents)
+        with open(filepath, "w") as f:
+          f.write(contents)
+
+if sys.version_info[0] < 3:
+    raise Exception("Python 3 or greater is required. Try running `python3 download_collection.py`")
+
+owner_name = ''
+private_token = ''
+
+# Read options
+optlist, args = getopt.getopt(sys.argv[1:], 'o:t:')
+
+for o, v in optlist:
+    if o == "-o":
+        owner_name = v.replace(" ", "%20")
+    if o == "-t":
+        private_token = v
+
+if not owner_name:
+    print('Error: missing `-o <owner_name>` option')
+    quit()
+
+if not private_token:
+    print('Error: missing `-t <private_token>` option')
+    quit()
+
+print("Updating models from {}.".format(owner_name))
+
+page = 1
+count = 0
+
+# The Fuel server URL.
+base_url ='https://fuel.gazebosim.org/'
+
+# Fuel server version.
+fuel_version = '1.0'
+
+# Path to get the models in the collection
+next_url = '/{}/models?page={}&per_page=100'.format(owner_name, page)
+
+# Path to download a single model in the collection
+download_url = base_url + fuel_version + '/{}/models/'.format(owner_name)
+
+download_dir = "dl"
+if not os.path.exists(download_dir):
+    os.makedirs(download_dir)
+
+# Iterate over the pages
+while True:
+    url = base_url + fuel_version + next_url
+
+    # Get the contents of the current page.
+    r = requests.get(url)
+
+    if not r or not r.text:
+        break
+
+    # Convert to JSON
+    models = json.loads(r.text)
+
+    # Compute the next page's URL
+    page = page + 1
+    next_url = '/{}/models?page={}&per_page=100'.format(owner_name, page)
+
+    # Download each model 
+    for model in models:
+        count+=1
+        model_name = model['name']
+        print ('Downloading (%d) %s' % (count, model_name))
+        download = requests.get(download_url+model_name+'.zip', stream=True)
+        file = model_name + '.zip'
+        dl_file = os.path.join(download_dir, file)
+        model_path = os.path.abspath(os.path.join(download_dir, model_name))
+
+        # Download the zip file
+        with open(dl_file, 'wb') as fd:
+            for chunk in download.iter_content(chunk_size=1024*1024):
+                fd.write(chunk)
+        # Extract the zip file
+        with zipfile.ZipFile(dl_file, 'r') as zip_ref:
+            zip_ref.extractall(model_path)
+        # Update plugin names
+        find_and_replace(model_path, "ignition::gazebo::systems", "gz::sim::systems", "sdf")
+        find_and_replace(model_path, r"(?:lib)?ignition-gazebo-([^.\s]*)(?:\.so)?", r"gz-sim-\1", "sdf")
+        find_and_replace(model_path, "ignition::gazebo::systems", "gz::sim::systems", "erb")
+        find_and_replace(model_path, r"(?:lib)?ignition-gazebo-([^.\s]*)(?:\.so)?", r"gz-sim-\1", "erb")
+
+        # Update fuel server
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "sdf")
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "config")
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "pbtxt")
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "dae")
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "obj")
+        find_and_replace(model_path, "fuel.ignitionrobotics", "fuel.gazebosim", "mtl")
+        model_url = '{}/{}/{}/models/{}'.format(base_url, fuel_version, owner_name, model_name)
+        token_header = 'Private-Token: {}'.format(private_token)
+        print("Uploading model {}".format(model_name))
+        subprocess.call(['gz', 'fuel', 'edit', '-u', model_url, '-o', owner_name, '--header', token_header, '-m', model_path])
+
+
+print('Done.')