Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert model:// to Fuel URLs instead of absolute paths #85

Merged
merged 8 commits into from
Jul 29, 2020
Merged

Convert model:// to Fuel URLs instead of absolute paths #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5f21270
Convert model:// to Fuel URLs instead of absolute paths
chapulina Jul 17, 2020
ed4f788
Backport file fetching
chapulina Jul 17, 2020
9e63e14
Convert model:// to Fuel URLs instead of absolute paths
chapulina Jul 17, 2020
99d54dd
remove extra /
chapulina Jul 18, 2020
15fbd67
fixed conflicts
chapulina Jul 18, 2020
b5858f2
merged from ign-fuel-tools3
chapulina Jul 22, 2020
6e9c320
codecheck
chapulina Jul 22, 2020
05bf741
PR feedback
chapulina Jul 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions src/FuelClient_TEST.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,10 +473,10 @@ TEST_F(FuelClientTest, DownloadModel)
"/test_cache/fuel.ignitionrobotics.org/iche033/models/Rescue Randy/2");
EXPECT_TRUE(common::exists(
"test_cache/fuel.ignitionrobotics.org/iche033/models/Rescue Randy/2"));
const std::string model_sdf_path =
const std::string modelSdfPath =
"test_cache/fuel.ignitionrobotics.org/iche033/models/Rescue Randy/2/"
"model.sdf";
EXPECT_TRUE(common::exists(model_sdf_path));
EXPECT_TRUE(common::exists(modelSdfPath));
EXPECT_TRUE(common::exists(
"test_cache/fuel.ignitionrobotics.org/iche033/models/Rescue Randy/2/"
"model.config"));
Expand All @@ -494,14 +494,21 @@ TEST_F(FuelClientTest, DownloadModel)
"/test_cache/fuel.ignitionrobotics.org/iche033/models/Rescue Randy/2",
cachedPath);

// Check that pbr paths have been updated.
std::ifstream ifs(model_sdf_path);
std::string model_sdf((std::istreambuf_iterator<char>(ifs)),
// Check that URIs have been updated.
std::ifstream ifs(modelSdfPath);
std::string modelSdf((std::istreambuf_iterator<char>(ifs)),
std::istreambuf_iterator<char>());
EXPECT_EQ(std::string::npos, model_sdf.find("<albedo_map>model://"));
EXPECT_EQ(std::string::npos, model_sdf.find("<normal_map>model://"));
EXPECT_EQ(std::string::npos, model_sdf.find("<metalness_map>model://"));
EXPECT_EQ(std::string::npos, model_sdf.find("<roughness_map>model://"));
EXPECT_EQ(std::string::npos, modelSdf.find("<uri>model://"));
EXPECT_EQ(std::string::npos, modelSdf.find("<albedo_map>model://"));
EXPECT_EQ(std::string::npos, modelSdf.find("<normal_map>model://"));
EXPECT_EQ(std::string::npos, modelSdf.find("<metalness_map>model://"));
EXPECT_EQ(std::string::npos, modelSdf.find("<roughness_map>model://"));

EXPECT_NE(std::string::npos, modelSdf.find("<uri>https://"));
EXPECT_NE(std::string::npos, modelSdf.find("<albedo_map>https://"));
EXPECT_NE(std::string::npos, modelSdf.find("<normal_map>https://"));
EXPECT_NE(std::string::npos, modelSdf.find("<metalness_map>https://"));
EXPECT_NE(std::string::npos, modelSdf.find("<roughness_map>https://"));
}

// Try using nonexistent URL
Expand Down
106 changes: 66 additions & 40 deletions src/LocalCache.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,34 +57,36 @@ class ignition::fuel_tools::LocalCachePrivate
/// \brief return all models in a given Owner/models directory
public: std::vector<Model> ModelsInPath(const std::string &_path);

/// \brief Associate model:// URI paths with paths on disk
/// \brief Associate model:// URI paths with Fuel URLs.
/// \param[in] _modelVersionedDir Directory containing the model.
/// \param[in] _id Model's Fuel URL.
/// \return True if the paths were fixed. False could occur if the
/// `model.config` file is not present or contains XML errors.
public: bool FixPaths(const std::string &_modelVersionedDir);
public: bool FixPaths(const std::string &_modelVersionedDir,
const ModelIdentifier &_id);

/// \brief Helper function to fix model:// URI paths in geometry elements.
/// \param[in] _geomElem Pointer to the geometry element.
/// \param[in] _modelVersionedDir Directory containing the model.
/// \param[in] _id Model identifier
/// \sa FixPaths
public: void FixPathsInGeomElement(tinyxml2::XMLElement *_geomElem,
const std::string &_modelVersionedDir);
const ModelIdentifier &_id);

/// \brief Helper function to fix model:// URI paths in material elements.
/// \param[in] _matElem Pointer to the material element.
/// \param[in] _modelVersionedDir Directory containing the model.
/// \param[in] _id Model identifier
/// \sa FixPaths
public: void FixPathsInMaterialElement(
tinyxml2::XMLElement *_matElem,
const std::string &_modelVersionedDir);
const ModelIdentifier &_id);

/// \brief Helper function to fix a single model:// URI that is contained
/// in an element.
/// \param[in] _elem Pointer to an element tha contains a URI.
/// \param[in] _modelVersionedDir Directory containing the model.
/// \param[in] _id Model identifier
/// \sa FixPaths
public: void FixPathsInUri(tinyxml2::XMLElement *_elem,
const std::string &_modelVersionedDir);
const ModelIdentifier &_id);

/// \brief client configuration
public: const ClientConfig *config = nullptr;
Expand Down Expand Up @@ -413,8 +415,8 @@ bool LocalCache::SaveModel(
return false;
}

// Convert model:// URIs to locations on disk.
this->dataPtr->FixPaths(modelVersionedDir);
// Convert model:// URIs to Fuel URLs
this->dataPtr->FixPaths(modelVersionedDir, _id);

// Cleanup the zip file.
if (!common::removeDirectoryOrFile(zipFile))
Expand All @@ -426,7 +428,8 @@ bool LocalCache::SaveModel(
}

//////////////////////////////////////////////////
bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)
bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir,
const ModelIdentifier &_id)
{
// Get model.config
std::string modelConfigPath = common::joinPaths(
Expand Down Expand Up @@ -498,7 +501,7 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)
while (collisionElem)
{
this->FixPathsInGeomElement(
collisionElem->FirstChildElement("geometry"), _modelVersionedDir);
collisionElem->FirstChildElement("geometry"), _id);
// Next collision element.
collisionElem = collisionElem->NextSiblingElement("collision");
}
Expand All @@ -509,9 +512,9 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)
while (visualElem)
{
this->FixPathsInGeomElement(
visualElem->FirstChildElement("geometry"), _modelVersionedDir);
visualElem->FirstChildElement("geometry"), _id);
this->FixPathsInMaterialElement(
visualElem->FirstChildElement("material"), _modelVersionedDir);
visualElem->FirstChildElement("material"), _id);
visualElem = visualElem->NextSiblingElement("visual");
}
linkElem = linkElem->NextSiblingElement("link");
Expand All @@ -531,7 +534,7 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)
auto filenameElem = skinElem->FirstChildElement("filename");
if (filenameElem)
{
this->FixPathsInUri(filenameElem, _modelVersionedDir);
this->FixPathsInUri(filenameElem, _id);
}
skinElem = skinElem->NextSiblingElement("skin");
}
Expand All @@ -543,7 +546,7 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)
auto filenameElem = animationElem->FirstChildElement("filename");
if (filenameElem)
{
this->FixPathsInUri(filenameElem, _modelVersionedDir);
this->FixPathsInUri(filenameElem, _id);
}
animationElem = animationElem->NextSiblingElement("animation");
}
Expand All @@ -556,34 +559,57 @@ bool LocalCachePrivate::FixPaths(const std::string &_modelVersionedDir)

//////////////////////////////////////////////////
void LocalCachePrivate::FixPathsInUri(tinyxml2::XMLElement *_elem,
const std::string &_modelVersionedDir)
const ModelIdentifier &_id)
{
if (!_elem)
return;

std::string uri = _elem->GetText();
std::string oldUri = _elem->GetText();
std::string prefix = "model://";

// Make sure the URI is of the form model://
if (uri.find(prefix) != std::string::npos)
{
int firstSlash = uri.find('/', prefix.size()+1);
std::string suffix = uri.substr(firstSlash);
if (oldUri.find(prefix) == std::string::npos)
return;

// Convert the model:// to point to an actual file.
// \todo(nkoenig) Handle URIs to other models. For
// example, ModelA may use something from ModelB.
std::string diskPath = common::joinPaths("file:/",
_modelVersionedDir, suffix);
int firstSlash = oldUri.find('/', prefix.size()+1);

_elem->SetText(diskPath.c_str());
auto resourceName = oldUri.substr(prefix.size(), firstSlash - prefix.size());

if (resourceName != _id.Name())
{
// On Blueprint and Citadel, just warn the user
// From Dome, use the name on the URI (resourceName) and assume the same
// owner
igndbg << "Model [" << _id.Name()
<< "] loading resource from another model, named [" << resourceName
<< "]. On Blueprint (ign-fuel-tools 3) and Citadel "
<< "(ign-fuel-tools 4), [" << resourceName << "] is ignored. "
<< "From Dome (ign-fuel-tools 5), [" << _id.Name()
<< "] will be used. If [" << resourceName
<< "] is not a model belonging to owner [" << _id.Owner()
<< "], fix your SDF file!" << std::endl;
}

auto filePath = oldUri.substr(firstSlash);

// Construct a model file URL used to download from the server
auto fuelUrl =
chapulina marked this conversation as resolved.
Show resolved Hide resolved
_id.Server().Url().Str() + '/' +
_id.Server().Version() + '/' +
_id.Owner() +
"/models/" +
_id.Name() + '/' +
_id.VersionStr() +
"/files" +
filePath;

_elem->SetText(fuelUrl.c_str());
}

//////////////////////////////////////////////////
void LocalCachePrivate::FixPathsInMaterialElement(
tinyxml2::XMLElement *_matElem,
const std::string &_modelVersionedDir)
const ModelIdentifier &_id)
{
if (!_matElem)
return;
Expand All @@ -597,7 +623,7 @@ void LocalCachePrivate::FixPathsInMaterialElement(
// Convert the "model://" URI pattern to file://
while (uriElem)
{
this->FixPathsInUri(uriElem, _modelVersionedDir);
this->FixPathsInUri(uriElem, _id);
uriElem = uriElem->NextSiblingElement("uri");
}
}
Expand All @@ -616,42 +642,42 @@ void LocalCachePrivate::FixPathsInMaterialElement(
tinyxml2::XMLElement *albedoElem =
workflowElem->FirstChildElement("albedo_map");
if (albedoElem)
this->FixPathsInUri(albedoElem, _modelVersionedDir);
this->FixPathsInUri(albedoElem, _id);
tinyxml2::XMLElement *normalElem =
workflowElem->FirstChildElement("normal_map");
if (normalElem)
this->FixPathsInUri(normalElem, _modelVersionedDir);
this->FixPathsInUri(normalElem, _id);
tinyxml2::XMLElement *envElem =
workflowElem->FirstChildElement("environment_map");
if (envElem)
this->FixPathsInUri(envElem, _modelVersionedDir);
this->FixPathsInUri(envElem, _id);
tinyxml2::XMLElement *emissiveElem =
workflowElem->FirstChildElement("emissive_map");
if (emissiveElem)
this->FixPathsInUri(emissiveElem, _modelVersionedDir);
this->FixPathsInUri(emissiveElem, _id);
// metal workflow specific elements
if (workflow == "metal")
{
tinyxml2::XMLElement *metalnessElem =
workflowElem->FirstChildElement("metalness_map");
if (metalnessElem)
this->FixPathsInUri(metalnessElem, _modelVersionedDir);
this->FixPathsInUri(metalnessElem, _id);
tinyxml2::XMLElement *roughnessElem =
workflowElem->FirstChildElement("roughness_map");
if (roughnessElem)
this->FixPathsInUri(roughnessElem, _modelVersionedDir);
this->FixPathsInUri(roughnessElem, _id);
}
// specular workflow specific elements
else if (workflow == "specular")
{
tinyxml2::XMLElement *specularElem =
workflowElem->FirstChildElement("specular_map");
if (specularElem)
this->FixPathsInUri(specularElem, _modelVersionedDir);
this->FixPathsInUri(specularElem, _id);
tinyxml2::XMLElement *glossinessElem =
workflowElem->FirstChildElement("glossiness_map");
if (glossinessElem)
this->FixPathsInUri(glossinessElem, _modelVersionedDir);
this->FixPathsInUri(glossinessElem, _id);
}
}
}
Expand All @@ -661,7 +687,7 @@ void LocalCachePrivate::FixPathsInMaterialElement(

//////////////////////////////////////////////////
void LocalCachePrivate::FixPathsInGeomElement(tinyxml2::XMLElement *_geomElem,
const std::string &_modelVersionedDir)
const ModelIdentifier &_id)
{
if (!_geomElem)
return;
Expand All @@ -673,7 +699,7 @@ void LocalCachePrivate::FixPathsInGeomElement(tinyxml2::XMLElement *_geomElem,
{
tinyxml2::XMLElement *uriElem = meshElem->FirstChildElement("uri");
// Convert the "model://" URI pattern to file://
this->FixPathsInUri(uriElem, _modelVersionedDir);
this->FixPathsInUri(uriElem, _id);
}
}

Expand Down