TestLifetimeWatcher: Address race condition in test assertions (#15969)

- If the timing is correct, a delay in the test's select might see the
   doneCh signal before the renew channels signal. If that happens, the
   test fails as it assumes we will receive signals across different
   channels in order.
 - Rework the test to make sure that we read from the renew channel if expected
   and the done channel so that any errors might not be escaping from detection
   on a renew.

api/renewer_test.go

@@ -192,6 +192,10 @@ func TestLifetimeWatcher(t *testing.T) {
 			}()
 			defer v.Stop()
 
+			receivedRenewal := false
+			receivedDone := false
+		ChannelLoop:
+			for {
 				select {
 				case <-time.After(tc.maxTestTime):
 					t.Fatalf("renewal didn't happen")
@@ -202,16 +206,29 @@ func TestLifetimeWatcher(t *testing.T) {
 					if r.Secret != renewedSecret {
 						t.Fatalf("expected secret %v, got %v", renewedSecret, r.Secret)
 					}
+					receivedRenewal = true
+					if !receivedDone {
+						continue ChannelLoop
+					}
+					break ChannelLoop
 				case err := <-doneCh:
+					receivedDone = true
 					if tc.expectError != nil && !errors.Is(err, tc.expectError) {
 						t.Fatalf("expected error %q, got: %v", tc.expectError, err)
 					}
 					if tc.expectError == nil && err != nil {
 						t.Fatalf("expected no error, got: %v", err)
 					}
-				if tc.expectRenewal {
+					if tc.expectRenewal && !receivedRenewal {
-					t.Fatalf("expected at least one renewal, got donech result: %v", err)
+						// We might have received the stop before the renew call on the channel.
+						continue ChannelLoop
 					}
+					break ChannelLoop
+				}
+			}
+
+			if tc.expectRenewal && !receivedRenewal {
+				t.Fatalf("expected at least one renewal, got none.")
 			}
 		})
 	}