Commit b4043367 authored by Christopher Ferris's avatar Christopher Ferris Committed by Gerrit Code Review
Browse files

Merge "[FAB-4901]: Harden delivery service unit tests."

parents dc7c40b7 e70ab84d
......@@ -104,7 +104,7 @@ type blocksProviderImpl struct {
const wrongStatusThreshold = 10
var maxRetryDelay = time.Second * 10
var MaxRetryDelay = time.Second * 10
var logger *logging.Logger // package-level logger
......@@ -152,7 +152,7 @@ func (b *blocksProviderImpl) DeliverBlocks() {
errorStatusCounter = 0
logger.Warningf("[%s] Got error %v", b.chainID, t)
}
maxDelay := float64(maxRetryDelay)
maxDelay := float64(MaxRetryDelay)
currDelay := float64(time.Duration(math.Pow(2, float64(statusCounter))) * 100 * time.Millisecond)
time.Sleep(time.Duration(math.Min(maxDelay, currDelay)))
if currDelay < maxDelay {
......
......@@ -32,7 +32,7 @@ import (
)
func init() {
maxRetryDelay = time.Second
MaxRetryDelay = time.Second
}
type mockMCS struct {
......
......@@ -19,6 +19,7 @@ package deliverclient
import (
"context"
"errors"
"fmt"
"runtime"
"sync"
"sync/atomic"
......@@ -250,6 +251,9 @@ func TestDeliverServiceFailover(t *testing.T) {
}
func TestDeliverServiceServiceUnavailable(t *testing.T) {
orgMaxRetryDelay := blocksprovider.MaxRetryDelay
blocksprovider.MaxRetryDelay = time.Millisecond * 200
defer func() { blocksprovider.MaxRetryDelay = orgMaxRetryDelay }()
defer ensureNoGoroutineLeak(t)()
// Scenario: bring up 2 ordering service instances,
// Make the instance the client connects to fail after a delivery of a block and send SERVICE_UNAVAILABLE
......@@ -292,6 +296,10 @@ func TestDeliverServiceServiceUnavailable(t *testing.T) {
activeInstance, backupInstance := waitForConnectionToSomeOSN()
assert.NotNil(t, activeInstance)
assert.NotNil(t, backupInstance)
// Check that delivery client get connected to active
assert.Equal(t, activeInstance.ConnCount(), 1)
// and not connected to backup instances
assert.Equal(t, backupInstance.ConnCount(), 0)
// Send first block
go activeInstance.SendBlock(li.Height)
......@@ -306,7 +314,7 @@ func TestDeliverServiceServiceUnavailable(t *testing.T) {
// Fail instance delivery client connected to
activeInstance.Fail()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
wg := sync.WaitGroup{}
......@@ -314,18 +322,22 @@ func TestDeliverServiceServiceUnavailable(t *testing.T) {
go func(ctx context.Context) {
defer wg.Done()
select {
case <-time.After(time.Millisecond * 100):
if backupInstance.ConnCount() > 0 {
for {
select {
case <-time.After(time.Millisecond * 100):
if backupInstance.ConnCount() > 0 {
return
}
case <-ctx.Done():
return
}
case <-ctx.Done():
return
}
}(ctx)
wg.Wait()
assert.NoError(t, ctx.Err(), "Delivery client has not failed over to alive ordering service")
// Check that delivery client was indeed connected
assert.Equal(t, backupInstance.ConnCount(), 1)
// Ensure the client asks blocks from the other ordering service node
assertBlockDissemination(li.Height, gossipServiceAdapter.GossipBlockDisseminations, t)
......@@ -452,7 +464,8 @@ func assertBlockDissemination(expectedSeq uint64, ch chan uint64, t *testing.T)
case seq := <-ch:
assert.Equal(t, expectedSeq, seq)
case <-time.After(time.Second * 5):
assert.Fail(t, "Didn't gossip a new block within a timely manner")
assert.FailNow(t, fmt.Sprintf("Didn't gossip a new block with seq num %d within a timely manner", expectedSeq))
t.Fatal()
}
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment