Refactor read_file and read_wfile: handle large files efficiently…

include/daw/daw_cfile_ptr.h

@@ -31,6 +31,44 @@ namespace daw {
 
 	[[nodiscard]] inline unique_file_ptr
 	open_cfile( daw::string_view const path, char const *DAW_RESTRICT modes ) {
-		return unique_file_ptr( std::fopen( path.get_c_str( ).c_str( ), modes ) );
+#if defined( _MSC_VER )
+		FILE *f = nullptr;
+		auto err = fopen_s( &f, path.get_c_str( ).c_str( ), modes );
+		if( err or not f ) {
+			return unique_file_ptr{ };
+		}
+#else
+		FILE *f = fopen( path.get_c_str( ).c_str( ), modes );
+		if( not f ) {
+			return unique_file_ptr{ };
+		}
+#endif
+		return unique_file_ptr( f );
 	}
+#if defined( _MSC_VER )
+	namespace wcfile_impl {
+		struct wcfile_deleter {
+			wcfile_deleter( ) = default;
+
+			DAW_CPP23_STATIC_CALL_OP inline void
+			operator( )( std::FILE *fp ) DAW_CPP23_STATIC_CALL_OP_CONST noexcept {
+				std::fclose( fp );
+			}
+		};
+	} // namespace wcfile_impl
+
+	using unique_wfile_ptr =
+	  daw::unique_ptr<std::FILE, wcfile_impl::wcfile_deleter>;
+
+	[[nodiscard]] inline unique_wfile_ptr
+	open_wcfile( daw::wstring_view const path,
+	             wchar_t const *DAW_RESTRICT modes ) {
+		FILE *f = nullptr;
+		auto err = _wfopen_s( &f, path.get_c_str( ).c_str( ), modes );
+		if( err or not f ) {
+			return unique_wfile_ptr{ };
+		}
+		return unique_wfile_ptr( f );
+	}
+#endif
 } // namespace daw

include/daw/daw_cpp_feature_check.h

@@ -264,4 +264,10 @@ inline constexpr bool daw_has_cx_cmath = false;
 #if __cpp_lib_containers_ranges >= 202202L
 #define DAW_HAS_CPP23_FROM_RANGE 1
 #endif
-#endif
+#endif
+
+#if defined( __cpp_lib_string_resize_and_overwrite )
+#if __cpp_lib_string_resize_and_overwrite >= 202110L
+#define DAW_HAS_CPP23_STR_RESIZE_OVERWRITE 1
+#endif
+#endif

include/daw/daw_read_file.h

@@ -9,59 +9,93 @@
 #pragma once
 
 #include "daw/ciso646.h"
+#include "daw/daw_as.h"
 #include "daw/daw_attributes.h"
+#include "daw/daw_cfile_ptr.h"
+#include "daw/daw_cpp_feature_check.h"
+#include "daw/daw_int_cmp.h"
 #include "daw/daw_string_view.h"
 #include "daw/daw_traits.h"
+#include "daw/daw_utility.h"
 
 #include <cstddef>
 #include <cstdio>
 #include <exception>
 #include <filesystem>
 #include <iostream>
+#include <iterator>
 #include <optional>
 #include <string>
 #include <string_view>
+#include <system_error>
 #include <utility>
 
 namespace daw {
 	template<typename CharT = char>
 	DAW_ATTRIB_NOINLINE std::optional<std::basic_string<CharT>>
 	read_file( daw::string_view path ) {
 		auto ec = std::error_code{ };
-		auto const fsize =
+		auto const fsize_tmp =
 		  std::filesystem::file_size( std::string_view( path ), ec );
 		if( ec ) {
 			return std::nullopt;
 		}
+		auto const fsize = daw::narrow_cast<std::size_t>( fsize_tmp );
 		if( fsize % sizeof( CharT ) != 0 ) {
 			return std::nullopt;
 		}
-		auto const element_count = fsize / sizeof( CharT );
-		if( static_cast<std::uintmax_t>( std::basic_string<CharT>{ }.max_size( ) ) <
-		    element_count ) {
+		auto element_count = fsize / sizeof( CharT ) > 0 ? fsize / sizeof( CharT )
+		                                                 : std::size_t{ 4096 };
+
+		if( daw::cmp_less( std::basic_string<CharT>{ }.max_size( ),
+		                   element_count ) ) {
 			// File is too big to fit into string(WIN32)
 			return std::nullopt;
 		}
-		auto result = std::basic_string<CharT>(
-		  static_cast<std::size_t>( element_count ), CharT{ } );
-#if defined( _MSC_VER )
-		FILE *f = nullptr;
-		auto err = fopen_s( &f, path.get_c_str( ).c_str( ), "rb" );
-		if( err or not f ) {
+		auto const f = daw::open_cfile( path, "rb" );
+		if( not f ) {
 			return std::nullopt;
 		}
+		auto result = std::basic_string<CharT>( );
+		bool keep_going = true;
+#if defined( DAW_HAS_CPP23_STR_RESIZE_OVERWRITE )
+		while( keep_going ) {
+			auto const old_size = result.size( );
+			result.resize_and_overwrite(
+			  old_size + element_count, [&]( CharT *p, std::size_t ) {
+				  std::advance( p, as<std::ptrdiff_t>( old_size ) );
+				  auto const num_read =
+				    std::fread( p, sizeof( CharT ), element_count, f.get( ) );
+				  keep_going = num_read == element_count;
+				  return old_size + num_read;
+			  } );
+			if( ferror( f.get( ) ) ) {
+				return std::nullopt;
+			}
+			if( keep_going ) {
+				// Ensure we don't expand too much and read efficiently
+				element_count = std::size_t{ 4096 };
+			}
+		}
 #else
-		auto *f = fopen( path.get_c_str( ).c_str( ), "rb" );
-		if( not f ) {
-			return std::nullopt;
+		while( keep_going ) {
+			auto const old_size = result.size( );
+			result.resize( old_size + element_count, CharT{ } );
+			CharT *p = std::next( result.data( ), as<std::ptrdiff_t>( old_size ) );
+			auto const num_read =
+			  std::fread( p, sizeof( CharT ), element_count, f.get( ) );
+			if( ferror( f.get( ) ) ) {
+				return std::nullopt;
+			}
+			keep_going = num_read == element_count;
+			if( keep_going ) {
+				// Ensure we don't expand too much and read efficiently
+				element_count = std::size_t{ 4096 };
+			} else {
+				result.resize( old_size + num_read );
+			}
 		}
 #endif
-		auto const num_read =
-		  fread( result.data( ), sizeof( CharT ), result.size( ), f );
-		auto const close_result = fclose( f );
-		if( num_read != result.size( ) or close_result != 0 ) {
-			return std::nullopt;
-		}
 		return result;
 	}
 
@@ -84,33 +118,67 @@ namespace daw {
 	read_wfile( daw::wstring_view path ) {
 		using CharT = wchar_t;
 		auto ec = std::error_code{ };
-		auto const fsize =
+		auto const fsize_tmp =
 		  std::filesystem::file_size( std::wstring_view( path ), ec );
 		if( ec ) {
 			return std::nullopt;
 		}
+		auto const fsize = daw::narrow_cast<std::size_t>( fsize_tmp );
 		if( fsize % sizeof( CharT ) != 0 ) {
 			return std::nullopt;
 		}
-		auto const element_count = fsize / sizeof( CharT );
-		if( static_cast<std::uintmax_t>( std::basic_string<CharT>{ }.max_size( ) ) <
-		    element_count ) {
+		auto element_count = fsize / sizeof( CharT ) > 0 ? fsize / sizeof( CharT )
+		                                                 : std::size_t{ 4096 };
+
+		if( daw::cmp_less( std::basic_string<CharT>{ }.max_size( ),
+		                   element_count ) ) {
 			// File is too big to fit into string(WIN32)
 			return std::nullopt;
 		}
-		auto result = std::basic_string<CharT>(
-		  static_cast<std::size_t>( element_count ), CharT{ } );
-		FILE *f = nullptr;
-		auto err = _wfopen_s( &f, path.get_c_str( ).c_str( ), L"rb" );
-		if( err or not f ) {
+		auto const f = daw::open_wcfile( path, L"rb" );
+		if( not f ) {
 			return std::nullopt;
 		}
-		auto const num_read =
-		  fread( result.data( ), sizeof( CharT ), result.size( ), f );
-		auto const close_result = fclose( f );
-		if( num_read != result.size( ) or close_result != 0 ) {
-			return std::nullopt;
+		auto result = std::basic_string<CharT>( );
+		bool keep_going = true;
+#if defined( DAW_HAS_CPP23_STR_RESIZE_OVERWRITE )
+		while( keep_going ) {
+			auto const old_size = result.size( );
+			result.resize_and_overwrite(
+			  old_size + element_count, [&]( CharT *p, std::size_t ) {
+				  std::advance( p, as<std::ptrdiff_t>( old_size ) );
+				  auto const num_read =
+				    std::fread( p, sizeof( CharT ), element_count, f.get( ) );
+				  keep_going = num_read == element_count;
+				  return old_size + num_read;
+			  } );
+			if( ferror( f.get( ) ) ) {
+				return std::nullopt;
+			}
+			if( keep_going ) {
+				// Ensure we don't expand too much and read efficiently
+				element_count = std::size_t{ 4096 };
+			}
 		}
+#else
+		while( keep_going ) {
+			auto const old_size = result.size( );
+			result.resize( old_size + element_count, CharT{ } );
+			CharT *p = std::next( result.data( ), as<std::ptrdiff_t>( old_size ) );
+			auto const num_read =
+			  std::fread( p, sizeof( CharT ), element_count, f.get( ) );
+			if( ferror( f.get( ) ) ) {
+				return std::nullopt;
+			}
+			keep_going = num_read == element_count;
+			if( keep_going ) {
+				// Ensure we don't expand too much and read efficiently
+				element_count = std::size_t{ 4096 };
+			} else {
+				result.resize( old_size + num_read );
+			}
+		}
+#endif
 		return result;
 	}
 
@@ -124,4 +192,5 @@ namespace daw {
 		return *result;
 	}
 #endif
+
 } // namespace daw

tests/CMakeLists.txt

@@ -28,6 +28,7 @@ set( TEST_SOURCES
 		 daw_attributes_test.cpp
 		 daw_benchmark_test.cpp
 		 daw_bounded_vector_test.cpp
+		 daw_cfile_ptr_test.cpp
 		 daw_constant_test.cpp
 		 daw_container_algorithm_test.cpp
 		 daw_contract_test.cpp
@@ -227,7 +228,6 @@ set( CPP20_NOT_MSVC_TEST_SOURCES
 
 set( NOT_MSVC_TEST_SOURCES
 		 daw_can_constant_evaluate_test.cpp
-		 daw_cfile_ptr_test.cpp
 		 daw_parser_helper_test.cpp
 		 daw_piecewise_factory_test.cpp
 		 daw_tuple2_test.cpp