Unverified Commit 082a9102 authored by Jay Guo's avatar Jay Guo Committed by Artem Barger
Browse files

[FAB-11863] Clean orderer network failure logs



Every failed attempt to send Step request is logged at ERROR level,
which pollutes etcdraft orderer leader logs when a follower is down.

This CR changes it to log at DEBUG level, except for the first failed
attempt and the first successful delivery after failure(s).

Test done: manually run cft integration test and inspect logs.

Change-Id: I1dd3468de1f6745f658c15e83a5e644e0b0492d6
Signed-off-by: default avatarJay Guo <guojiannan1101@gmail.com>
parent 2ad9d9da
......@@ -813,6 +813,11 @@ func (c *Chain) isConfig(env *common.Envelope) bool {
}
func (c *Chain) configureComm() error {
// Reset unreachable map when communication is reconfigured
c.node.unreachableLock.Lock()
c.node.unreachable = make(map[uint64]struct{})
c.node.unreachableLock.Unlock()
nodes, err := c.remotePeers()
if err != nil {
return err
......
......@@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
package etcdraft
import (
"sync"
"time"
"code.cloudfoundry.org/clock"
......@@ -22,6 +23,9 @@ type node struct {
chainID string
logger *flogging.FabricLogger
unreachableLock sync.RWMutex
unreachable map[uint64]struct{}
storage *RaftStorage
config *raft.Config
......@@ -96,6 +100,9 @@ func (n *node) run() {
}
func (n *node) send(msgs []raftpb.Message) {
n.unreachableLock.RLock()
defer n.unreachableLock.RUnlock()
for _, msg := range msgs {
if msg.To == 0 {
continue
......@@ -107,9 +114,12 @@ func (n *node) send(msgs []raftpb.Message) {
_, err := n.rpc.Step(msg.To, &orderer.StepRequest{Channel: n.chainID, Payload: msgBytes})
if err != nil {
// TODO We should call ReportUnreachable if message delivery fails
n.logger.Errorf("Failed to send StepRequest to %d, because: %s", msg.To, err)
n.logSendFailure(msg.To, err)
status = raft.SnapshotFailure
} else if _, ok := n.unreachable[msg.To]; ok {
n.logger.Infof("Successfully sent StepRequest to %d after failed attempt(s)", msg.To)
delete(n.unreachable, msg.To)
}
if msg.Type == raftpb.MsgSnap {
......@@ -118,6 +128,16 @@ func (n *node) send(msgs []raftpb.Message) {
}
}
func (n *node) logSendFailure(dest uint64, err error) {
if _, ok := n.unreachable[dest]; ok {
n.logger.Debugf("Failed to send StepRequest to %d, because: %s", dest, err)
return
}
n.logger.Errorf("Failed to send StepRequest to %d, because: %s", dest, err)
n.unreachable[dest] = struct{}{}
}
func (n *node) takeSnapshot(index uint64, cs raftpb.ConfState, data []byte) {
if err := n.storage.TakeSnapshot(index, cs, data); err != nil {
n.logger.Errorf("Failed to create snapshot at index %d: %s", index, err)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment