Fix #2648 - Studio: shapes imported after "Save As" fail with "Error reading"

Applications/shapeworks/shapeworks.cpp

@@ -6,11 +6,14 @@
 #include "Executable.h"
 #include "Commands.h"
 #include <Profiling.h>
+#include <ShapeworksUtils.h>
 
 using namespace shapeworks;
 
 int main(int argc, char *argv[])
 {
+  ShapeWorksUtils::initialize_path_handling();
+
   Executable shapeworks;
 
   // Image Commands

Libs/Analyze/Shape.cpp

@@ -5,6 +5,7 @@
 #include <Particles/ParticleSystemEvaluation.h>
 #include <Project/ProjectUtils.h>
 #include <Shape.h>
+#include <ShapeworksUtils.h>
 #include <Utils/StringUtils.h>
 #include <itkImageFileReader.h>
 #include <itkOrientImageFilter.h>
@@ -912,7 +913,7 @@ void Shape::set_reconstruction_transforms(std::vector<vtkSmartPointer<vtkTransfo
 
 //---------------------------------------------------------------------------
 void Shape::load_feature_from_scalar_file(std::string filename, std::string feature_name) {
-  if (!boost::filesystem::exists(filename)) {
+  if (!ShapeWorksUtils::file_exists(filename)) {
     return;
   }
 

Libs/Common/ShapeworksUtils.cpp

@@ -1,11 +1,19 @@
+// boost::nowide's utf8_codecvt is header-only, but including it pulls in the auto-link pragma
+// that asks MSVC for boost_nowide.lib, which the dependency build does not produce.  This must
+// precede every Boost header.
+#define BOOST_NOWIDE_NO_LIB
+
 #include "ShapeworksUtils.h"
 
 #include <Logging.h>
 #include <sys/stat.h>
 #include <tbb/global_control.h>
 #include <tbb/info.h>
+#include <vtksys/SystemTools.hxx>
 
 #include <boost/filesystem.hpp>
+#include <boost/nowide/utf8_codecvt.hpp>
+#include <locale>
 
 namespace shapeworks {
 
@@ -19,16 +27,33 @@ void ShapeWorksUtils::set_rng_seed(const unsigned seed) {
   mt_.seed(rng_seed_);
 }
 
+//-----------------------------------------------------------------------------
+void ShapeWorksUtils::initialize_path_handling() {
+  // boost::filesystem stores a path in the character type the OS API uses: char on POSIX, but
+  // wchar_t on Windows.  There every narrow string is converted using the codecvt facet imbued
+  // into boost::filesystem::path, which by default comes from the global locale rather than being
+  // UTF-8.  The rest of ShapeWorks (Qt, VTK, ITK) speaks UTF-8, so without this a path holding
+  // non-ASCII characters is mangled on the way in and no longer resolves.  On POSIX no conversion
+  // takes place and this only affects the wide-string accessors.
+  boost::filesystem::path::imbue(std::locale(std::locale(), new boost::nowide::utf8_codecvt<wchar_t>));
+}
+
 //-----------------------------------------------------------------------------
 bool ShapeWorksUtils::is_directory(const std::string& pathname) {
-  boost::system::error_code ec;
-  return boost::filesystem::is_directory(pathname, ec);
+  return vtksys::SystemTools::FileIsDirectory(pathname);
 }
 
 //-----------------------------------------------------------------------------
 bool ShapeWorksUtils::file_exists(const std::string& filename) {
-  boost::system::error_code ec;
-  return boost::filesystem::is_regular_file(filename, ec);
+  // vtksys rather than boost::filesystem: on Windows it decodes UTF-8 and applies the \\?\ prefix,
+  // so it agrees with the VTK and ITK readers about which paths exist.  boost::filesystem does
+  // neither, and rejects long or non-ASCII paths that those readers open without complaint.
+  return vtksys::SystemTools::FileExists(filename, true);
+}
+
+//-----------------------------------------------------------------------------
+bool ShapeWorksUtils::path_exists(const std::string& pathname) {
+  return vtksys::SystemTools::PathExists(pathname);
 }
 
 //-----------------------------------------------------------------------------

Libs/Common/ShapeworksUtils.h

@@ -31,14 +31,19 @@ class ShapeWorksUtils {
   /// generates random number
   static unsigned get_random_number() { return mt_(); }
 
+  //! Configure process-wide filesystem path handling.  Call this once at startup, before any path
+  //! is constructed from a narrow string.
+  static void initialize_path_handling();
+
   /// returns true if pathname is a directory
-  // TODO: in C++17 this is a standard function
   static bool is_directory(const std::string& pathname);
 
-  /// returns true if filename exists
-  // TODO: in C++17 this is a standard function
+  /// returns true if filename exists and can be read
   static bool file_exists(const std::string& filename);
 
+  /// returns true if a file or a directory exists at pathname
+  static bool path_exists(const std::string& pathname);
+
   //! Set up the console logging options
   static void setup_console_logging(bool show_progress, bool xml_status);
 

Libs/Image/Image.cpp

@@ -117,7 +117,7 @@ Image::ImageType::Pointer Image::read(const std::string& pathname) {
   }
 
   // check if it exists
-  if (!boost::filesystem::exists(pathname)) {
+  if (!ShapeWorksUtils::file_exists(pathname)) {
     throw std::runtime_error("File does not exist: " + pathname);
   }
 

Libs/Project/Project.cpp

@@ -1,6 +1,7 @@
 #include <Logging.h>
 #include <Mesh/MeshUtils.h>
 #include <Project.h>
+#include <ShapeworksUtils.h>
 #include <StringUtils.h>
 #include <vtkPointData.h>
 
@@ -91,17 +92,27 @@ void Project::set_project_path(const std::string& new_pathname) {
   auto old_path = fs::path(project_path_);
   auto new_path = fs::path(new_pathname);
 
+  // Re-anchor one path to the new project location.  Returns the path unchanged if it can't be,
+  // so that a single awkward path doesn't fail the whole save.
+  auto rebase = [&](std::string path) {
+    if (!ShapeWorksUtils::path_exists(path)) {
+      return path;
+    }
+    // replace \ with / in path
+    path = StringUtils::replace_string(path, "\\", "/");
+    try {
+      auto canonical = fs::canonical(path, old_path);
+      return fs::relative(canonical, new_path).string();
+    } catch (std::exception& e) {
+      SW_WARN("Unable to update path \"{}\" for new project location: {}", path, e.what());
+      return path;
+    }
+  };
+
   auto fixup = [&](StringList paths) {
     StringList new_paths;
-    for (auto path : paths) {
-      if (fs::exists(path)) {
-        // replace \ with / in path
-        path = StringUtils::replace_string(path, "\\", "/");
-        auto canonical = fs::canonical(path, old_path);
-        new_paths.push_back(fs::relative(canonical, new_path).string());
-      } else {
-        new_paths.push_back(path);
-      }
+    for (auto& path : paths) {
+      new_paths.push_back(rebase(path));
     }
     return new_paths;
   };
@@ -117,11 +128,7 @@ void Project::set_project_path(const std::string& new_pathname) {
     auto features = subject->get_feature_filenames();
     project::types::StringMap new_features;
     for (auto const& x : features) {
-      auto path = x.second;
-      // replace \ with / in path
-      path = StringUtils::replace_string(path, "\\", "/");
-      auto canonical = fs::canonical(path, old_path);
-      new_features[x.first] = fs::relative(canonical, new_path).string();
+      new_features[x.first] = rebase(x.second);
     }
     subject->set_feature_filenames(new_features);
   }
@@ -350,7 +357,7 @@ void Project::determine_feature_names() {
     if (get_original_domain_types()[d] == DomainType::Mesh) {
       if (subject->get_original_filenames().size() > d) {
         auto filename = subject->get_original_filenames()[d];
-        if (fs::exists(filename)) {
+        if (ShapeWorksUtils::file_exists(filename)) {
           try {
             auto poly_data = MeshUtils::threadSafeReadMesh(filename).getVTKMesh();
             if (poly_data) {

Libs/Python/ShapeworksPython.cpp

@@ -59,6 +59,8 @@ namespace fs = boost::filesystem;
 using namespace shapeworks;
 
 PYBIND11_MODULE(shapeworks_py, m) {
+  ShapeWorksUtils::initialize_path_handling();
+
   m.doc() = "ShapeWorks Python API";
 
   m.attr("Pi") = std::atan(1.0) * 4.0;

Studio/main.cpp

@@ -19,6 +19,7 @@
 // vtk
 #include <Applications/Configuration.h>
 #include <Interface/ShapeWorksStudioApp.h>
+#include <ShapeworksUtils.h>
 #include <Utils/PlatformUtils.h>
 #include <Logging.h>
 #include <QVTKOpenGLNativeWidget.h>
@@ -83,6 +84,8 @@ int main(int argc, char** argv) {
 
   TIME_SCOPE("ShapeWorksStudio");
   try {
+    ShapeWorksUtils::initialize_path_handling();
+
 #ifdef Q_OS_MACOS
     // Prevent cursor crashes on Apple Silicon
     qputenv("QT_MAC_DISABLE_NATIVE_CURSORS", "1");

Testing/CMakeLists.txt

@@ -55,6 +55,7 @@ target_include_directories(Testing PUBLIC
 target_link_libraries(Testing
   gtest
   gtest_main
+  Common
   ${Boost_LIBRARIES}
   )
 set_target_properties(Testing PROPERTIES PUBLIC_HEADER

Testing/MeshTests/MeshTests.cpp

@@ -1,6 +1,7 @@
 #include <igl/point_mesh_squared_distance.h>
 #include <vtkCellArray.h>
 #include <vtkPoints.h>
+#include <vtksys/SystemTools.hxx>
 
 #include <chrono>
 #include <fstream>
@@ -11,6 +12,7 @@
 #include "MeshWarper.h"
 #include "ParticleSystem.h"
 #include "ParticleSystemEvaluation.h"
+#include "ShapeworksUtils.h"
 #include "Testing.h"
 
 using namespace shapeworks;
@@ -1163,3 +1165,38 @@ TEST(MeshTests, extractLargestComponentSingleComponentTest) {
   ASSERT_EQ(femur.numPoints(), original_points);
   ASSERT_EQ(femur.numFaces(), original_faces);
 }
+
+TEST(MeshTests, nonAsciiPathTest) {
+  // Mesh's existence check has to agree with the reader underneath it about which paths are
+  // readable.  On Windows boost::filesystem did not, and meshes that VTK opened without trouble
+  // were rejected as missing before the reader ever saw them. (#2648)
+  auto dir = TestUtils::Instance().get_output_dir("non_ascii_path_test");
+  std::string unicode_dir = dir + "/Ünïcode Dätä";
+  ASSERT_TRUE(vtksys::SystemTools::MakeDirectory(unicode_dir));
+
+  std::string path = unicode_dir + "/mesh.vtk";
+  Mesh(std::string(TEST_DATA_DIR) + "/femur.vtk").write(path);
+
+  ASSERT_TRUE(ShapeWorksUtils::file_exists(path));
+  ASSERT_NO_THROW(Mesh mesh(path));
+}
+
+TEST(MeshTests, longPathTest) {
+  // Paths beyond MAX_PATH need the \\?\ prefix on Windows.  The STL reader applies it; the
+  // existence check in front of it did not. (#2648)  This uses .stl deliberately: VTK's legacy .vtk
+  // reader stats the file without the prefix first, so it can't open such a path on Windows at all.
+  auto dir = TestUtils::Instance().get_output_dir("long_path_test");
+
+  std::string deep = dir;
+  while (deep.size() < 300) {
+    deep += "/a_directory_with_a_deliberately_long_name";
+  }
+  ASSERT_TRUE(vtksys::SystemTools::MakeDirectory(deep));
+
+  std::string path = deep + "/mesh.stl";
+  ASSERT_GT(path.size(), 260u);
+  Mesh(std::string(TEST_DATA_DIR) + "/femur.vtk").write(path);
+
+  ASSERT_TRUE(ShapeWorksUtils::file_exists(path));
+  ASSERT_NO_THROW(Mesh mesh(path));
+}

Testing/Testing.cpp

@@ -1,5 +1,7 @@
 #include "Testing.h"
 
+#include <ShapeworksUtils.h>
+
 #include <boost/filesystem.hpp>
 #include <cstdlib>
 
@@ -31,7 +33,7 @@ TestUtils& TestUtils::Instance() {
 }
 
 //-----------------------------------------------------------------------------
-TestUtils::TestUtils() {}
+TestUtils::TestUtils() { ShapeWorksUtils::initialize_path_handling(); }
 
 //-----------------------------------------------------------------------------
 TestUtils::~TestUtils() {