Merge branch 'feature/library_intermediate' into feature/library_intermediate2

README.md

@@ -6,11 +6,31 @@ See [https://www.cryfs.org](https://www.cryfs.org).
 Install latest release
 ======================
 
+Linux
+------
+
 This only works for Ubuntu 17.04 and later, and Debian Stretch and later.
 You can also use CryFS on older versions of these distributions by following the **Building from source** instructions below.
 
     sudo apt install cryfs
     
+OSX
+----
+
+CryFS is distributed via Homebrew. Just do
+
+    brew cask install osxfuse
+    brew install cryfs
+    
+Windows (experimental)
+----------------------
+
+CryFS has experimental Windows support since the 0.10 release series. To install it, do:
+
+1. Install [DokanY](https://github.com/dokan-dev/dokany/releases)
+2. Install [Microsoft Visual C++ Redistributable for Visual Studio 2017](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads)
+3. Install [CryFS](https://www.cryfs.org/#download)
+
 GUI
 ===
 Theres some GUI applications with CryFS support. You usually have to install the GUI **and** also CryFS itself for it to work.
@@ -72,13 +92,15 @@ You can pass the following variables to the *cmake* command (using *-Dvariablena
  - **-DCRYFS_UPDATE_CHECKS**=off: Build a CryFS that doesn't check online for updates and security vulnerabilities.
 
 Building on Windows (experimental)
----------------
+----------------------------------
 
 Build with Visual Studio 2017 and pass in the following flags to CMake:
 
     -DDOKAN_PATH=[dokan library location, e.g. "C:\Program Files\Dokan\DokanLibrary-1.1.0"]
     -DBOOST_ROOT=[path to root of boost installation]
 
+If you set these variables correctly in the `CMakeSettings.json` file, you should be able to open the cryfs source folder with Visual Studio 2017.
+
 Troubleshooting
 ---------------
 
@@ -141,7 +163,7 @@ There are additional requirements if you want to create packages. They are:
  2. Build
 
         $ mkdir cmake && cd cmake
-        $ cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=off
+        $ cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_TESTING=off
         $ make package
 
 

appveyor.yml

@@ -2,7 +2,7 @@ image:
 #- Visual Studio 2013
 #- Visual Studio 2015
 - Visual Studio 2017
-- Visual Studio 2017 Preview
+#- Visual Studio 2017 Preview
 
 platform:
 - x64

src/cryfs/config/CryConfigLoader.cpp

@@ -71,7 +71,7 @@ void CryConfigLoader::_checkVersion(const CryConfig &config, bool allowFilesyste
     }
   }
   if (!allowFilesystemUpgrade && gitversion::VersionCompare::isOlderThan(config.Version(), CryConfig::FilesystemFormatVersion)) {
-    if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It can be migrated, but afterwards couldn't be opened anymore with older versions. Please make a backup of your data before attempting a migration. Do you want to migrate it now?", false)) {
+    if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It is recommended to create a new filesystem with CryFS 0.10 and copy your files into it. If you don't want to do that, we can also attempt to migrate the existing filesystem, but that can take a long time, you won't be getting some of the performance advantages of the 0.10 release series, and if the migration fails, your data may be lost. If you decide to continue, please make sure you have a backup of your data. Do you want to attempt a migration now?", false)) {
       throw CryfsException("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). It has to be migrated.", ErrorCode::TooOldFilesystemFormat);
     }
   }

src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp

@@ -56,7 +56,7 @@ boost::optional<unique_ref<FsBlob>> FsBlobStore::load(const blockstore::BlockId
             dir.AppendChildrenTo(&children);
             for (const auto &child : children) {
                 auto childEntry = dir.GetChild(child.name);
-                ASSERT(childEntry != none, "Couldn't load child, although it was returned as a child in the lsit.");
+                ASSERT(childEntry != none, "Couldn't load child, although it was returned as a child in the list.");
                 auto childBlob = _baseBlobStore->load(childEntry->blockId());
                 ASSERT(childBlob != none, "Couldn't load child blob");
                 _migrate(std::move(*childBlob), dir.blockId());

test/cryfs/config/CryConfigLoaderTest.cpp

@@ -310,7 +310,7 @@ TEST_F(CryConfigLoaderTest, AsksWhenLoadingNewerFilesystem_AnswerNo) {
 }
 
 TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(true));
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to attempt a migration now?"), false)).Times(1).WillOnce(Return(true));
 
     string version = olderVersion();
     CreateWithVersion(version, version);
@@ -318,14 +318,14 @@ TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
 }
 
 TEST_F(CryConfigLoaderTest, DoesNotAskForMigrationWhenCorrectVersion) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), _)).Times(0);
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to attempt a migration now?"), _)).Times(0);
 
     CreateWithVersion(gitversion::VersionString(), CryConfig::FilesystemFormatVersion);
     EXPECT_NE(boost::none, Load());
 }
 
 TEST_F(CryConfigLoaderTest, DontMigrateWhenAnsweredNo) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(false));
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to attempt a migration now?"), false)).Times(1).WillOnce(Return(false));
 
     string version = olderVersion();
     CreateWithVersion(version, version);

vendor/README

@@ -5,3 +5,8 @@ spdlog: https://github.com/gabime/spdlog/tree/v0.16.3/include/spdlog
 cryptopp: https://github.com/weidai11/cryptopp/tree/CRYPTOPP_8_0_0
  - changed: added CMakeLists.txt and cryptopp-config.cmake from https://github.com/noloader/cryptopp-cmake/tree/CRYPTOPP_8_0_0
  - changed: commented out line including winapifamily.h in CMakeLists.txt
+ - cherry-picked commits to get OpenMP for scrypt on Windows:
+   - https://github.com/weidai11/cryptopp/commit/aa043b38a7930725c31a0cd7016986d1c581c573
+   - https://github.com/weidai11/cryptopp/commit/672f5c7f3dad8ae12b2d0ce0940ccb7c8e257bf8
+   - https://github.com/weidai11/cryptopp/commit/7e96a283a3192d29aac5b60e5b4ff19248f00d9a
+   - https://github.com/weidai11/cryptopp/commit/ca32b63038d5f7b13e2e00809cd9184a1efe8c24

vendor/cryptopp/CMakeLists.txt

@@ -9,9 +9,6 @@ target_compile_definitions(cryptopp PUBLIC $<$<CONFIG:Debug>:CRYPTOPP_DEBUG>) #
 add_compile_options($<$<CONFIG:Debug>:-DCRYPTOPP_DEBUG>) # add to stuff built in subdirectories (like the actual library)
 
 if(NOT DISABLE_OPENMP)
-    if (MSVC)
-        message(WARNING "MSVC does not support the OpenMP 4.0 standard used by Crypto++. Disabling OpenMP. This can cause degraded performance.")
-    else()
     find_package(OpenMP)
 
     if (OPENMP_FOUND OR OPENMP_CXX_FOUND)
@@ -78,7 +75,6 @@ if(NOT DISABLE_OPENMP)
     message(STATUS "OpenMP flags: ${OpenMP_CXX_FLAGS}")
     string(REPLACE " " ";" REPLACED_FLAGS ${OpenMP_CXX_FLAGS})
     add_compile_options(${REPLACED_FLAGS})
-    endif()
 else()
     message(WARNING "OpenMP is disabled. This can cause degraded performance.")
 endif()

vendor/cryptopp/vendor_cryptopp/cryptest.nmake

@@ -139,6 +139,7 @@ LDLIBS =
 # CXXFLAGS = $(CXXFLAGS) /DDEBUG /D_DEBUG /Oi /Oy- /Od /MTd
 # Release build. Add /OPT:REF to linker
 CXXFLAGS = $(CXXFLAGS) /DNDEBUG /D_NDEBUG /Oi /Oy /O2 /MT
+# Linker flags.
 LDFLAGS = $(LDFLAGS) /OPT:REF
 
 # Attempt to detect when <sdkddkver.h> and <winapifamily.h> are available

vendor/cryptopp/vendor_cryptopp/salsa.cpp

@@ -90,9 +90,14 @@ void Salsa20_Core(word32* data, unsigned int rounds)
 		x[15] ^= rotlConstant<18>(x[14]+x[13]);
 	}
 
+#ifdef _MSC_VER
+	for (size_t i = 0; i < 16; ++i)
+		data[i] += x[i];
+#else
 	#pragma omp simd
 	for (size_t i = 0; i < 16; ++i)
 		data[i] += x[i];
+#endif
 }
 
 std::string Salsa20_Policy::AlgorithmProvider() const

vendor/cryptopp/vendor_cryptopp/scrypt.cpp

@@ -14,6 +14,8 @@
 #include "sha.h"
 
 #include <sstream>
+#include <limits>
+
 #ifdef _OPENMP
 # include <omp.h>
 #endif
@@ -53,9 +55,14 @@ static inline void BlockCopy(byte* dest, byte* src, size_t len)
 
 static inline void BlockXOR(byte* dest, byte* src, size_t len)
 {
+#ifdef _MSC_VER
+    for (size_t i = 0; i < len; ++i)
+        dest[i] ^= src[i];
+#else
     #pragma omp simd
     for (size_t i = 0; i < len; ++i)
         dest[i] ^= src[i];
+#endif
 }
 
 static inline void PBKDF2_SHA256(byte* buf, size_t dkLen,
@@ -171,6 +178,16 @@ void Scrypt::ValidateParameters(size_t derivedLen, word64 cost, word64 blockSize
         }
     }
 
+    // https://github.com/weidai11/cryptopp/issues/787
+    CRYPTOPP_ASSERT(parallelization <= std::numeric_limits<int>::max());
+    if (parallelization > static_cast<word64>(std::numeric_limits<int>::max()))
+    {
+        std::ostringstream oss;
+        oss << " parallelization " << parallelization << " is larger than ";
+        oss << std::numeric_limits<int>::max();
+        throw InvalidArgument("Scrypt: " + oss.str());
+    }
+
     CRYPTOPP_ASSERT(IsPowerOf2(cost));
     if (IsPowerOf2(cost) == false)
         throw InvalidArgument("Scrypt: cost must be a power of 2");
@@ -245,10 +262,13 @@ size_t Scrypt::DeriveKey(byte*derived, size_t derivedLen, const byte*secret, siz
     // 1: (B_0 ... B_{p-1}) <-- PBKDF2(P, S, 1, p * MFLen)
     PBKDF2_SHA256(B, B.size(), secret, secretLen, salt, saltLen, 1);
 
+    // Visual Studio and OpenMP 2.0 fixup. We must use int, not size_t.
+    int maxParallel=0;
+    if (!SafeConvert(parallel, maxParallel))
+        maxParallel = std::numeric_limits<int>::max();
+
     #ifdef _OPENMP
-    int threads = STDMIN(omp_get_max_threads(),
-        static_cast<int>(STDMIN(static_cast<size_t>(parallel),
-        static_cast<size_t>(std::numeric_limits<int>::max()))));
+    int threads = STDMIN(omp_get_max_threads(), maxParallel);
     #endif
 
     // http://stackoverflow.com/q/49604260/608639
@@ -260,7 +280,7 @@ size_t Scrypt::DeriveKey(byte*derived, size_t derivedLen, const byte*secret, siz
 
         // 2: for i = 0 to p - 1 do
         #pragma omp for
-        for (size_t i = 0; i < static_cast<size_t>(parallel); ++i)
+        for (int i = 0; i < maxParallel; ++i)
         {
             // 3: B_i <-- MF(B_i, N)
             const ptrdiff_t offset = static_cast<ptrdiff_t>(blockSize*i*128);

vendor/cryptopp/vendor_cryptopp/scrypt.h

@@ -76,7 +76,9 @@ public:
     /// \details The parameter <tt>blockSize</tt> ("r" in the documents) specifies the block
     ///   size.
     /// \details The <tt>parallelization</tt> parameter ("p" in the documents) is a positive
-    ///   integer less than or equal to <tt>((2^32-1) * 32) / (128 * r)</tt>.
+    ///   integer less than or equal to <tt>((2^32-1) * 32) / (128 * r)</tt>. Due to Microsoft
+    ///   and its OpenMP 2.0 implementation <tt>parallelization</tt> is limited to
+    ///   <tt>std::numeric_limits<int>::max()</tt>.
     /// \details Scrypt always returns 1 because it only performs 1 iteration. Other
     ///   derivation functions, like PBKDF's, will return more interesting values.
     /// \details The Crypto++ implementation of Scrypt is limited by C++ datatypes. For