2015-04-10 21:52:30 +02:00
# include "testutils/DataTreeTest.h"
# include "testutils/TwoLevelDataFixture.h"
2016-02-11 15:19:58 +01:00
# include "blobstore/implementations/onblocks/utils/Math.h"
# include <cpp-utils/data/Data.h>
2015-04-10 21:52:30 +02:00
# include <tuple>
using : : testing : : WithParamInterface ;
using : : testing : : Values ;
using : : testing : : Combine ;
using std : : tuple ;
using std : : get ;
using std : : function ;
using std : : mem_fn ;
using cpputils : : dynamic_pointer_move ;
using blobstore : : onblocks : : datanodestore : : DataLeafNode ;
using blobstore : : onblocks : : datanodestore : : DataInnerNode ;
using blobstore : : onblocks : : datanodestore : : DataNode ;
using blobstore : : onblocks : : datanodestore : : DataNodeLayout ;
using blobstore : : onblocks : : datatreestore : : DataTree ;
using blobstore : : onblocks : : utils : : ceilDivision ;
2017-09-17 03:07:27 +02:00
using blockstore : : BlockId ;
2015-04-25 02:55:34 +02:00
using cpputils : : Data ;
2015-06-26 15:59:18 +02:00
using boost : : none ;
2015-04-10 21:52:30 +02:00
2015-06-26 15:59:18 +02:00
using cpputils : : unique_ref ;
2015-04-10 21:52:30 +02:00
class DataTreeTest_ResizeByTraversing : public DataTreeTest {
public :
static constexpr DataNodeLayout LAYOUT = DataNodeLayout ( BLOCKSIZE_BYTES ) ;
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateTree ( unique_ref < DataNode > root ) {
2017-09-17 03:07:27 +02:00
BlockId blockId = root - > blockId ( ) ;
2015-07-21 15:24:49 +02:00
cpputils : : destruct ( std : : move ( root ) ) ;
2017-09-17 03:07:27 +02:00
return treeStore . load ( blockId ) . value ( ) ;
2015-04-10 21:52:30 +02:00
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateLeafTreeWithSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateLeafWithSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateTwoLeafTreeWithSecondLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateTwoLeafWithSecondLeafSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateFullTwoLevelTreeWithLastLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateFullTwoLevelWithLastLeafSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateThreeLevelTreeWithTwoChildrenAndLastLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateThreeLevelWithTwoChildrenAndLastLeafSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateThreeLevelTreeWithThreeChildrenAndLastLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateThreeLevelWithThreeChildrenAndLastLeafSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateFullThreeLevelTreeWithLastLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateFullThreeLevelWithLastLeafSize ( size ) ) ;
}
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > CreateFourLevelMinDataTreeWithLastLeafSize ( uint32_t size ) {
2015-04-10 21:52:30 +02:00
return CreateTree ( CreateFourLevelMinDataWithLastLeafSize ( size ) ) ;
}
2017-09-17 03:07:27 +02:00
void EXPECT_IS_LEFTMAXDATA_TREE ( const BlockId & blockId ) {
auto root = nodeStore - > load ( blockId ) . value ( ) ;
2015-04-10 21:52:30 +02:00
DataInnerNode * inner = dynamic_cast < DataInnerNode * > ( root . get ( ) ) ;
if ( inner ! = nullptr ) {
for ( uint32_t i = 0 ; i < inner - > numChildren ( ) - 1 ; + + i ) {
2017-12-03 20:01:41 +01:00
EXPECT_IS_MAXDATA_TREE ( inner - > readChild ( i ) . blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
2017-12-03 20:01:41 +01:00
EXPECT_IS_LEFTMAXDATA_TREE ( inner - > readLastChild ( ) . blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
}
2017-09-17 03:07:27 +02:00
void EXPECT_IS_MAXDATA_TREE ( const BlockId & blockId ) {
auto root = nodeStore - > load ( blockId ) . value ( ) ;
2015-04-10 21:52:30 +02:00
DataInnerNode * inner = dynamic_cast < DataInnerNode * > ( root . get ( ) ) ;
if ( inner ! = nullptr ) {
for ( uint32_t i = 0 ; i < inner - > numChildren ( ) ; + + i ) {
2017-12-03 20:01:41 +01:00
EXPECT_IS_MAXDATA_TREE ( inner - > readChild ( i ) . blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
} else {
DataLeafNode * leaf = dynamic_cast < DataLeafNode * > ( root . get ( ) ) ;
EXPECT_EQ ( nodeStore - > layout ( ) . maxBytesPerLeaf ( ) , leaf - > numBytes ( ) ) ;
}
}
} ;
constexpr DataNodeLayout DataTreeTest_ResizeByTraversing : : LAYOUT ;
2016-07-14 11:41:49 +02:00
class DataTreeTest_ResizeByTraversing_P : public DataTreeTest_ResizeByTraversing , public WithParamInterface < tuple < function < unique_ref < DataTree > ( DataTreeTest_ResizeByTraversing * , uint32_t ) > , uint32_t , uint32_t , std : : function < uint32_t ( uint32_t oldNumberOfLeaves , uint32_t newNumberOfLeaves ) > > > {
2015-04-10 21:52:30 +02:00
public :
DataTreeTest_ResizeByTraversing_P ( )
: oldLastLeafSize ( get < 1 > ( GetParam ( ) ) ) ,
tree ( get < 0 > ( GetParam ( ) ) ( this , oldLastLeafSize ) ) ,
numberOfLeavesToAdd ( get < 2 > ( GetParam ( ) ) ) ,
newNumberOfLeaves ( tree - > numLeaves ( ) + numberOfLeavesToAdd ) ,
2016-07-14 11:41:49 +02:00
traversalBeginIndex ( get < 3 > ( GetParam ( ) ) ( tree - > numLeaves ( ) , newNumberOfLeaves ) ) ,
2015-04-10 21:52:30 +02:00
ZEROES ( LAYOUT . maxBytesPerLeaf ( ) )
{
ZEROES . FillWithZeroes ( ) ;
}
2017-09-17 03:07:27 +02:00
void GrowTree ( const BlockId & blockId ) {
auto tree = treeStore . load ( blockId ) ;
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) . get ( ) ) ;
2015-04-10 21:52:30 +02:00
}
2019-01-13 08:21:12 +01:00
void GrowTree ( DataTree * tree ) {
2016-07-12 01:04:33 +02:00
uint64_t maxBytesPerLeaf = tree - > maxBytesPerLeaf ( ) ;
2019-01-13 08:21:12 +01:00
uint64_t offset = traversalBeginIndex * maxBytesPerLeaf ;
uint64_t count = newNumberOfLeaves * maxBytesPerLeaf - offset ;
Data data ( count ) ;
data . FillWithZeroes ( ) ;
tree - > writeBytes ( data . data ( ) , offset , count ) ;
2015-04-10 21:52:30 +02:00
tree - > flush ( ) ;
}
2017-09-17 03:07:27 +02:00
unique_ref < DataLeafNode > LastLeaf ( const BlockId & blockId ) {
auto root = nodeStore - > load ( blockId ) . value ( ) ;
2015-04-10 21:52:30 +02:00
auto leaf = dynamic_pointer_move < DataLeafNode > ( root ) ;
2015-06-26 15:59:18 +02:00
if ( leaf ! = none ) {
return std : : move ( * leaf ) ;
2015-04-10 21:52:30 +02:00
}
2015-06-28 16:45:18 +02:00
auto inner = dynamic_pointer_move < DataInnerNode > ( root ) . value ( ) ;
2017-12-03 20:01:41 +01:00
return LastLeaf ( inner - > readLastChild ( ) . blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
uint32_t oldLastLeafSize ;
2015-06-26 15:59:18 +02:00
unique_ref < DataTree > tree ;
2015-04-10 21:52:30 +02:00
uint32_t numberOfLeavesToAdd ;
uint32_t newNumberOfLeaves ;
2016-07-14 11:41:49 +02:00
uint32_t traversalBeginIndex ;
2015-04-10 21:52:30 +02:00
Data ZEROES ;
} ;
INSTANTIATE_TEST_CASE_P ( DataTreeTest_ResizeByTraversing_P , DataTreeTest_ResizeByTraversing_P ,
Combine (
//Tree we're starting with
2015-06-26 15:59:18 +02:00
Values < function < unique_ref < DataTree > ( DataTreeTest_ResizeByTraversing * , uint32_t ) > > (
2015-04-10 21:52:30 +02:00
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateLeafTreeWithSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateTwoLeafTreeWithSecondLeafSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateFullTwoLevelTreeWithLastLeafSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateThreeLevelTreeWithTwoChildrenAndLastLeafSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateThreeLevelTreeWithThreeChildrenAndLastLeafSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateFullThreeLevelTreeWithLastLeafSize ) ,
mem_fn ( & DataTreeTest_ResizeByTraversing : : CreateFourLevelMinDataTreeWithLastLeafSize )
) ,
//Last leaf size of the start tree
Values (
0u ,
1u ,
10u ,
DataTreeTest_ResizeByTraversing : : LAYOUT . maxBytesPerLeaf ( )
) ,
//Number of leaves we're adding
Values (
1u ,
2u ,
DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) , //Full two level tree
2 * DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) , //Three level tree with two children
3 * DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) , //Three level tree with three children
DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) * DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) , //Full three level tree
DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) * DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) + 1 //Four level mindata tree
2016-07-14 11:41:49 +02:00
) ,
//Decide the traversal begin index
Values (
[ ] ( uint32_t /*oldNumberOfLeaves*/ , uint32_t newNumberOfLeaves ) { return newNumberOfLeaves - 1 ; } , // Traverse last leaf (begin==end-1)
[ ] ( uint32_t oldNumberOfLeaves , uint32_t newNumberOfLeaves ) { return ( oldNumberOfLeaves + newNumberOfLeaves ) / 2 ; } , // Start traversal in middle of new leaves
[ ] ( uint32_t oldNumberOfLeaves , uint32_t /*newNumberOfLeaves*/ ) { return oldNumberOfLeaves - 1 ; } , // Start traversal with last old leaf
[ ] ( uint32_t oldNumberOfLeaves , uint32_t /*newNumberOfLeaves*/ ) { return oldNumberOfLeaves ; } , // Start traversal with first new leaf
[ ] ( uint32_t /*oldNumberOfLeaves*/ , uint32_t /*newNumberOfLeaves*/ ) { return 0 ; } , // Traverse full tree
[ ] ( uint32_t /*oldNumberOfLeaves*/ , uint32_t /*newNumberOfLeaves*/ ) { return 1 ; } // Traverse full tree except first leaf
2015-04-10 21:52:30 +02:00
)
)
) ;
TEST_P ( DataTreeTest_ResizeByTraversing_P , StructureIsValid ) {
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) ) ;
2017-09-17 03:07:27 +02:00
EXPECT_IS_LEFTMAXDATA_TREE ( tree - > blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
2016-07-06 08:56:44 +02:00
TEST_P ( DataTreeTest_ResizeByTraversing_P , NumLeavesIsCorrect_FromCache ) {
tree - > numLeaves ( ) ; // fill cache with old value
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) ) ;
2015-04-10 21:52:30 +02:00
// tree->numLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
// This is what the StructureIsValid test case is for.
EXPECT_EQ ( newNumberOfLeaves , tree - > numLeaves ( ) ) ;
}
2016-07-06 08:56:44 +02:00
TEST_P ( DataTreeTest_ResizeByTraversing_P , NumLeavesIsCorrect ) {
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) ) ;
2019-01-13 08:21:12 +01:00
// tree->forceComputeNumLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
2016-07-06 08:56:44 +02:00
// This is what the StructureIsValid test case is for.
2019-01-13 08:21:12 +01:00
EXPECT_EQ ( newNumberOfLeaves , tree - > forceComputeNumLeaves ( ) ) ;
2016-07-06 08:56:44 +02:00
}
2015-04-10 21:52:30 +02:00
TEST_P ( DataTreeTest_ResizeByTraversing_P , DepthFlagsAreCorrect ) {
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) ) ;
2015-04-10 21:52:30 +02:00
uint32_t depth = ceil ( log ( newNumberOfLeaves ) / log ( DataTreeTest_ResizeByTraversing : : LAYOUT . maxChildrenPerInnerNode ( ) ) ) ;
2017-09-17 03:07:27 +02:00
CHECK_DEPTH ( depth , tree - > blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
TEST_P ( DataTreeTest_ResizeByTraversing_P , KeyDoesntChange ) {
2017-09-17 03:07:27 +02:00
BlockId blockId = tree - > blockId ( ) ;
2015-04-10 21:52:30 +02:00
tree - > flush ( ) ;
2016-07-14 11:41:49 +02:00
GrowTree ( tree . get ( ) ) ;
2017-09-17 03:07:27 +02:00
EXPECT_EQ ( blockId , tree - > blockId ( ) ) ;
2015-04-10 21:52:30 +02:00
}
TEST_P ( DataTreeTest_ResizeByTraversing_P , DataStaysIntact ) {
2019-01-13 08:21:12 +01:00
uint32_t oldNumberOfLeaves = std : : max ( UINT64_C ( 1 ) , ceilDivision ( tree - > numBytes ( ) , static_cast < uint64_t > ( nodeStore - > layout ( ) . maxBytesPerLeaf ( ) ) ) ) ;
2015-04-10 21:52:30 +02:00
TwoLevelDataFixture data ( nodeStore , TwoLevelDataFixture : : SizePolicy : : Unchanged ) ;
2017-09-17 03:07:27 +02:00
BlockId blockId = tree - > blockId ( ) ;
2015-07-21 15:24:49 +02:00
cpputils : : destruct ( std : : move ( tree ) ) ;
2017-09-17 03:07:27 +02:00
data . FillInto ( nodeStore - > load ( blockId ) . get ( ) . get ( ) ) ;
2015-04-10 21:52:30 +02:00
2017-09-17 03:07:27 +02:00
GrowTree ( blockId ) ;
2015-04-10 21:52:30 +02:00
2019-01-13 08:21:12 +01:00
if ( traversalBeginIndex < oldNumberOfLeaves ) {
// Traversal wrote over part of the pre-existing data, we can only check the data before it.
if ( traversalBeginIndex ! = 0 ) {
data . EXPECT_DATA_CORRECT ( nodeStore - > load ( blockId ) . get ( ) . get ( ) , traversalBeginIndex - 1 ) ;
}
} else {
// Here, traversal was entirely outside the preexisting data, we can check all preexisting data.
data . EXPECT_DATA_CORRECT ( nodeStore - > load ( blockId ) . get ( ) . get ( ) , oldNumberOfLeaves , oldLastLeafSize ) ;
2016-09-07 19:02:41 +02:00
}
}