From fd07b66173620c57118db17a1832482b43b3f871 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sun, 16 Sep 2018 23:37:30 -0700 Subject: [PATCH] Remove curl dependency on Windows, use WinHttp instead --- CMakeSettings.json | 8 +- README.md | 2 - appveyor.yml | 3 +- src/cpp-utils/CMakeLists.txt | 12 +- src/cpp-utils/network/CurlHttpClient.cpp | 4 + src/cpp-utils/network/CurlHttpClient.h | 4 + src/cpp-utils/network/WinHttpClient.cpp | 258 +++++++++++++++++++++++ src/cpp-utils/network/WinHttpClient.h | 30 +++ src/cryfs-cli/main.cpp | 15 +- 9 files changed, 322 insertions(+), 14 deletions(-) create mode 100644 src/cpp-utils/network/WinHttpClient.cpp create mode 100644 src/cpp-utils/network/WinHttpClient.h diff --git a/CMakeSettings.json b/CMakeSettings.json index 88b97f2a..d1494502 100644 --- a/CMakeSettings.json +++ b/CMakeSettings.json @@ -7,7 +7,7 @@ "inheritEnvironments": [ "msvc_x86" ], "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", - "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\" -DCURL_LIBRARY=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\lib\\libcurl.dll.a -DCURL_INCLUDE_DIR=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\include", + "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\"", "buildCommandArgs": "-v", "ctestCommandArgs": "" }, @@ -18,7 +18,7 @@ "inheritEnvironments": [ "msvc_x86" ], "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", - "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\" -DCURL_LIBRARY=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\lib\\libcurl.dll.a -DCURL_INCLUDE_DIR=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\include", + "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\"", "buildCommandArgs": "-v", "ctestCommandArgs": "" }, @@ -29,7 +29,7 @@ "inheritEnvironments": [ "msvc_x64_x64" ], "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", - "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\" -DCURL_LIBRARY=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\lib\\libcurl.dll.a -DCURL_INCLUDE_DIR=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\include", + "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\"", "buildCommandArgs": "-v", "ctestCommandArgs": "" }, @@ -40,7 +40,7 @@ "inheritEnvironments": [ "msvc_x64_x64" ], "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", - "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\" -DCURL_LIBRARY=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\lib\\libcurl.dll.a -DCURL_INCLUDE_DIR=C:\\ProgramData\\chocolatey\\lib\\curl\\tools\\curl-7.61.0-win64-mingw\\include", + "cmakeCommandArgs": "-DBUILD_TESTING=on -DDOKAN_LIB_PATH=\"C:\\Program Files\\Dokan\\DokanLibrary-1.1.0\"", "buildCommandArgs": "-v", "ctestCommandArgs": "" } diff --git a/README.md b/README.md index f4e1f1aa..750d0bf3 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,6 @@ Building on Windows (experimental) Build with Visual Studio 2017 and pass in the following flags to CMake: -DDOKAN_LIB_PATH=[dokan library location, e.g. "C:\Program Files\Dokan\DokanLibrary-1.1.0"] - -DCURL_LIBRARY=[path to libcurl.dll.a, e.g. "C:\ProgramData\chocolatey\lib\curl\tools\curl-7.61.1-win64-mingw\lib\libcurl.dll.a"] - -DCURL_INCLUDE_DIR=[path to libcurl include files, e.g. "C:\ProgramData\chocolatey\lib\curl\tools\curl-7.61.1-win64-mingw\include"] -DBOOST_ROOT=[path to root of boost installation] Troubleshooting diff --git a/appveyor.yml b/appveyor.yml index 65abbc79..be160fb2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,14 +12,13 @@ configuration: - RelWithDebInfo install: - - choco install -y curl - choco install -y dokany --installargs ADDLOCAL=DokanDevFeature,DokanLibBFeature,DokanPDBFeature - cmake --version build_script: - cmd: mkdir build - cmd: cd build - - cmake .. -G "Visual Studio 15 2017 Win64" -DBUILD_TESTING=on -DBOOST_ROOT=C:\Libraries\boost_1_65_1 -DCURL_LIBRARY=C:\ProgramData\chocolatey\lib\curl\tools\curl-7.61.1-win64-mingw\lib\libcurl.dll.a -DCURL_INCLUDE_DIR=C:\ProgramData\chocolatey\lib\curl\tools\curl-7.61.1-win64-mingw\include -DDOKAN_LIB_PATH="C:\Program Files\Dokan\DokanLibrary-1.1.0" + - cmake .. -G "Visual Studio 15 2017 Win64" -DBUILD_TESTING=on -DBOOST_ROOT=C:\Libraries\boost_1_65_1 -DDOKAN_LIB_PATH="C:\Program Files\Dokan\DokanLibrary-1.1.0" # TODO Make build parallel - cmd: cmake --build . --config %CONFIGURATION% - cmd: .\test\gitversion\%CONFIGURATION%\gitversion-test.exe diff --git a/src/cpp-utils/CMakeLists.txt b/src/cpp-utils/CMakeLists.txt index d1e0451c..07ca7416 100644 --- a/src/cpp-utils/CMakeLists.txt +++ b/src/cpp-utils/CMakeLists.txt @@ -15,6 +15,7 @@ set(SOURCES tempfile/TempDir.cpp network/HttpClient.cpp network/CurlHttpClient.cpp + network/WinHttpClient.cpp network/FakeHttpClient.cpp io/Console.cpp io/DontEchoStdinToStdoutRAII.cpp @@ -54,9 +55,6 @@ set(SOURCES add_library(${PROJECT_NAME} STATIC ${SOURCES}) -find_package(CURL REQUIRED) -target_include_directories(${PROJECT_NAME} PUBLIC ${CURL_INCLUDE_DIRS}) -target_link_libraries(${PROJECT_NAME} PUBLIC ${CURL_LIBRARIES}) if(NOT MSVC) find_package(Backtrace REQUIRED) @@ -66,6 +64,14 @@ else() target_link_libraries(${PROJECT_NAME} PUBLIC DbgHelp) endif() +if (NOT MSVC) + find_package(CURL REQUIRED) + target_include_directories(${PROJECT_NAME} PUBLIC ${CURL_INCLUDE_DIRS}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${CURL_LIBRARIES}) +else() + target_link_libraries(${PROJECT_NAME} PUBLIC WinHttp) +endif() + find_package(Threads REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/src/cpp-utils/network/CurlHttpClient.cpp b/src/cpp-utils/network/CurlHttpClient.cpp index 57885cad..49903459 100644 --- a/src/cpp-utils/network/CurlHttpClient.cpp +++ b/src/cpp-utils/network/CurlHttpClient.cpp @@ -1,5 +1,7 @@ // Base version taken from https://techoverflow.net/blog/2013/03/15/c-simple-http-download-using-libcurl-easy-api/ +#if !defined(_MSC_VER) + #include "CurlHttpClient.h" #include #include @@ -67,3 +69,5 @@ namespace cpputils { } } + +#endif diff --git a/src/cpp-utils/network/CurlHttpClient.h b/src/cpp-utils/network/CurlHttpClient.h index 7c3f20d1..de9d37e7 100644 --- a/src/cpp-utils/network/CurlHttpClient.h +++ b/src/cpp-utils/network/CurlHttpClient.h @@ -2,6 +2,8 @@ #ifndef MESSMER_CPPUTILS_NETWORK_CURLHTTPCLIENT_HPP #define MESSMER_CPPUTILS_NETWORK_CURLHTTPCLIENT_HPP +#if !defined(_MSC_VER) + #include "HttpClient.h" #include "../macros.h" #include @@ -42,3 +44,5 @@ namespace cpputils { } #endif + +#endif diff --git a/src/cpp-utils/network/WinHttpClient.cpp b/src/cpp-utils/network/WinHttpClient.cpp new file mode 100644 index 00000000..995c3f0c --- /dev/null +++ b/src/cpp-utils/network/WinHttpClient.cpp @@ -0,0 +1,258 @@ +#if defined(_MSC_VER) + +#include "WinHttpClient.h" +#include +#include +#include +#include +#include +#include +#include + +using boost::none; +using boost::optional; +using std::string; +using std::wstring; +using std::wstring_convert; +using std::ostringstream; + +namespace cpputils { + + namespace { + struct HttpHandleRAII final { + HINTERNET handle; + + HttpHandleRAII(HINTERNET handle_) : handle(handle_) {} + + HttpHandleRAII(HttpHandleRAII&& rhs) : handle(rhs.handle) { + rhs.handle = nullptr; + } + + ~HttpHandleRAII() { + if (nullptr != handle) { + BOOL success = WinHttpCloseHandle(handle); + if (!success) { + throw std::runtime_error("Error calling WinHttpCloseHandle. Error code: " + std::to_string(GetLastError())); + } + } + } + + DISALLOW_COPY_AND_ASSIGN(HttpHandleRAII); + }; + + URL_COMPONENTS parse_url(const wstring &url) { + URL_COMPONENTS result; + result.dwStructSize = sizeof(result); + // Declare fields we want. Setting a field to nullptr and the length to non-zero means the field will be returned. + result.lpszScheme = nullptr; + result.dwSchemeLength = 1; + result.lpszHostName = nullptr; + result.dwHostNameLength = 1; + result.lpszUserName = nullptr; + result.dwUserNameLength = 1; + result.lpszPassword = nullptr; + result.dwPasswordLength = 1; + result.lpszUrlPath = nullptr; + result.dwUrlPathLength = 1; + result.lpszExtraInfo = nullptr; + result.dwExtraInfoLength = 1; + + BOOL success = WinHttpCrackUrl(url.c_str(), url.size(), ICU_REJECT_USERPWD, &result); + if (!success) { + throw std::runtime_error("Error parsing url '" + wstring_convert>().to_bytes(url) + "'. Error code: " + std::to_string(GetLastError())); + } + + return result; + } + + INTERNET_PORT get_port_from_url(const URL_COMPONENTS& parsedUrl) { + wstring scheme_str(parsedUrl.lpszScheme, parsedUrl.dwSchemeLength); + string s_(wstring_convert < std::codecvt_utf8_utf16>().to_bytes(scheme_str)); + if (parsedUrl.nScheme == INTERNET_SCHEME_HTTP) { + ASSERT(scheme_str == L"http", "Scheme mismatch"); + if (parsedUrl.nPort != 80) { + throw std::runtime_error("We don't support non-default ports"); + } + return INTERNET_DEFAULT_HTTP_PORT; + } + else if (parsedUrl.nScheme == INTERNET_SCHEME_HTTPS) { + ASSERT(scheme_str == L"https", "Scheme mismatch"); + if (parsedUrl.nPort != 443) { + throw std::runtime_error("We don't support non-default ports"); + } + return INTERNET_DEFAULT_HTTPS_PORT; + } + else { + throw std::runtime_error("Unsupported scheme: " + wstring_convert>().to_bytes(scheme_str)); + } + } + + class Request final { + public: + Request(HttpHandleRAII request) : request_(std::move(request)) {} + + void set_redirect_policy(DWORD redirectPolicy) { + BOOL success = WinHttpSetOption(request_.handle, WINHTTP_OPTION_REDIRECT_POLICY, &redirectPolicy, sizeof(redirectPolicy)); + if (!success) { + throw std::runtime_error("Error calling WinHttpSetOption. Error code: " + std::to_string(GetLastError())); + } + } + + void set_timeouts(long timeoutMsec) { + // TODO Timeout should be a total timeout, not per step as we're doing it here. + BOOL success = WinHttpSetTimeouts(request_.handle, timeoutMsec, timeoutMsec, timeoutMsec, timeoutMsec); + if (!success) { + throw std::runtime_error("Error calling WinHttpSetTimeouts. Error code: " + std::to_string(GetLastError())); + } + } + + void send() { + BOOL success = WinHttpSendRequest(request_.handle, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); + if (!success) { + throw std::runtime_error("Error calling WinHttpSendRequest. Error code: " + std::to_string(GetLastError())); + } + } + + void wait_for_response() { + BOOL success = WinHttpReceiveResponse(request_.handle, nullptr); + if (!success) { + throw std::runtime_error("Error calling WinHttpReceiveResponse. Error code: " + std::to_string(GetLastError())); + } + } + + DWORD get_status_code() { + DWORD statusCode; + DWORD statusCodeSize = sizeof(statusCode); + BOOL success = WinHttpQueryHeaders(request_.handle, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, WINHTTP_HEADER_NAME_BY_INDEX, &statusCode, &statusCodeSize, WINHTTP_NO_HEADER_INDEX); + if (!success) { + throw std::runtime_error("Eror calling WinHttpQueryHeaders. Error code: " + std::to_string(GetLastError())); + } + return statusCode; + } + + string read_response() { + ostringstream result; + + while (true) { + DWORD size = num_bytes_readable(); + if (size == 0) { + break; + } + + cpputils::Data buffer(size + 1); + buffer.FillWithZeroes(); + + DWORD num_read; + BOOL success = WinHttpReadData(request_.handle, buffer.data(), buffer.size(), &num_read); + if (!success) { + throw std::runtime_error("Error calling WinHttpReadData. Error code: " + std::to_string(GetLastError())); + } + ASSERT(0 != num_read, "Weird behavior of WinHttpReadData.It should never read zero bytes since WinHttpQueryDataAvailable said there are bytes readable."); + + result.write(reinterpret_cast(buffer.data()), num_read); + ASSERT(result.good(), "Error writing to ostringstream"); + } + + return result.str(); + } + + private: + DWORD num_bytes_readable() { + DWORD result; + BOOL success = WinHttpQueryDataAvailable(request_.handle, &result); + if (!success) { + throw std::runtime_error("Error calling WinHttpQueryDataAvailable. Error code: " + std::to_string(GetLastError())); + } + return result; + } + + HttpHandleRAII request_; + }; + + struct Connection final { + public: + Connection(HttpHandleRAII connection) : connection_(std::move(connection)) {} + + Request create_request(const URL_COMPONENTS& parsedUrl) { + const INTERNET_PORT port = get_port_from_url(parsedUrl); + const wstring path = wstring(parsedUrl.lpszUrlPath, parsedUrl.dwUrlPathLength) + wstring(parsedUrl.lpszExtraInfo, parsedUrl.dwExtraInfoLength); + const DWORD flags = (port == INTERNET_DEFAULT_HTTPS_PORT) ? WINHTTP_FLAG_SECURE : 0; + + HttpHandleRAII request_handle(WinHttpOpenRequest(connection_.handle, L"GET", path.c_str(), nullptr, WINHTTP_NO_REFERER, WINHTTP_DEFAULT_ACCEPT_TYPES, flags)); + if (nullptr == request_handle.handle) { + throw std::runtime_error("Error calling WinHttpOpenRequest. Error code: " + std::to_string(GetLastError())); + } + return Request(std::move(request_handle)); + } + + private: + HttpHandleRAII connection_; + }; + } + + struct WinHttpSession final { + public: + WinHttpSession(HttpHandleRAII session) : session_(std::move(session)) {} + + Connection create_connection(const URL_COMPONENTS& parsedUrl) { + const INTERNET_PORT port = get_port_from_url(parsedUrl); + const wstring host(parsedUrl.lpszHostName, parsedUrl.dwHostNameLength); + + HttpHandleRAII connection_handle = WinHttpConnect(session_.handle, host.c_str(), port, 0); + if (nullptr == connection_handle.handle) { + throw std::runtime_error("Error calling WinHttpConnect. Error code: " + std::to_string(GetLastError())); + } + + return Connection(std::move(connection_handle)); + } + + private: + HttpHandleRAII session_; + }; + + namespace { + std::unique_ptr create_session() { + HttpHandleRAII session_handle = WinHttpOpen(L"cpputils::HttpClient", WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); + if(nullptr == session_handle.handle) { + throw std::runtime_error("Error calling WinHttpOpen. Error code: " + std::to_string(GetLastError())); + } + + return std::make_unique(std::move(session_handle)); + } + } + + WinHttpClient::WinHttpClient() : session_(create_session()) {} + + WinHttpClient::~WinHttpClient() {} + + string WinHttpClient::get(const string &url, optional timeoutMsec) { + wstring wurl = wstring_convert>().from_bytes(url); + const URL_COMPONENTS parsedUrl = parse_url(wurl); + + ASSERT(parsedUrl.dwUserNameLength == 0, "Authentication not supported"); + ASSERT(parsedUrl.dwPasswordLength == 0, "Authentication not supported"); + + Connection connection = session_->create_connection(parsedUrl); + Request request = connection.create_request(parsedUrl); + + // allow redirects but not from https to http + request.set_redirect_policy(WINHTTP_OPTION_REDIRECT_POLICY_DISALLOW_HTTPS_TO_HTTP); + + if (timeoutMsec != none) { + request.set_timeouts(*timeoutMsec); + } + + request.send(); + request.wait_for_response(); + + DWORD statusCode = request.get_status_code(); + if (statusCode != HTTP_STATUS_OK) { + throw std::runtime_error("HTTP Server returned unsupported status code: " + std::to_string(statusCode)); + } + + return request.read_response(); + } + +} + +#endif diff --git a/src/cpp-utils/network/WinHttpClient.h b/src/cpp-utils/network/WinHttpClient.h new file mode 100644 index 00000000..f84517bd --- /dev/null +++ b/src/cpp-utils/network/WinHttpClient.h @@ -0,0 +1,30 @@ +#pragma once +#ifndef MESSMER_CPPUTILS_NETWORK_WINHTTPCLIENT_HPP +#define MESSMER_CPPUTILS_NETWORK_WINHTTPCLIENT_HPP + +#if defined(_MSC_VER) + +#include "HttpClient.h" +#include "../macros.h" + +namespace cpputils { + + class WinHttpSession; + + class WinHttpClient final : public HttpClient { + public: + WinHttpClient(); + ~WinHttpClient(); + + std::string get(const std::string &url, boost::optional timeoutMsec = boost::none) override; + + private: + std::unique_ptr session_; + + DISALLOW_COPY_AND_ASSIGN(WinHttpClient); + }; + +} + +#endif +#endif diff --git a/src/cryfs-cli/main.cpp b/src/cryfs-cli/main.cpp index 56232096..66068362 100644 --- a/src/cryfs-cli/main.cpp +++ b/src/cryfs-cli/main.cpp @@ -1,14 +1,18 @@ #include "Cli.h" #include #include -#include #include #include +#if defined(_MSC_VER) +#include +#else +#include +#endif + using namespace cryfs; using cpputils::Random; using cpputils::SCrypt; -using cpputils::CurlHttpClient; using cpputils::IOStreamConsole; using cpputils::make_unique_ref; using std::make_shared; @@ -17,8 +21,13 @@ using std::cerr; int main(int argc, const char *argv[]) { try { auto &keyGenerator = Random::OSRandom(); +#if defined(_MSC_VER) + auto httpClient = make_unique_ref(); +#else + auto httpClient = make_unique_ref(); +#endif return Cli(keyGenerator, SCrypt::DefaultSettings, make_shared()) - .main(argc, argv, make_unique_ref()); + .main(argc, argv, std::move(httpClient)); } catch (const CryfsException &e) { if (e.errorCode() != ErrorCode::Success) { std::cerr << "Error: " << e.what() << std::endl;