retry_test.go 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. /*
  2. Copyright © The ESO Authors
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. https://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package grpc
  14. import (
  15. "context"
  16. "errors"
  17. "fmt"
  18. "testing"
  19. "time"
  20. "google.golang.org/grpc/codes"
  21. "google.golang.org/grpc/status"
  22. )
  23. func TestCircuitBreaker_States(t *testing.T) {
  24. cb := NewCircuitBreaker(CircuitBreakerConfig{
  25. MaxFailures: 3,
  26. Timeout: 100 * time.Millisecond,
  27. HalfOpenMaxRequests: 1,
  28. })
  29. ctx := context.Background()
  30. // Initially closed
  31. if cb.State() != StateClosed {
  32. t.Errorf("expected initial state to be Closed, got %v", cb.State())
  33. }
  34. // Simulate 3 failures to open the circuit
  35. for range 3 {
  36. err := cb.Call(ctx, func() error {
  37. return status.Error(codes.Unavailable, "service unavailable")
  38. })
  39. if err == nil {
  40. t.Fatal("expected error from failing call")
  41. }
  42. }
  43. // Should now be open
  44. if cb.State() != StateOpen {
  45. t.Errorf("expected state to be Open after failures, got %v", cb.State())
  46. }
  47. // Requests should fail fast
  48. err := cb.Call(ctx, func() error {
  49. t.Fatal("should not execute function in open state")
  50. return nil
  51. })
  52. if err == nil || err.Error() != "circuit breaker is open" {
  53. t.Errorf("expected circuit open error, got: %v", err)
  54. }
  55. // Wait for timeout to transition to half-open
  56. time.Sleep(150 * time.Millisecond)
  57. // Should now be half-open
  58. err = cb.Call(ctx, func() error {
  59. return nil // Success
  60. })
  61. if err != nil {
  62. t.Fatalf("expected success in half-open state, got: %v", err)
  63. }
  64. // Should transition back to closed
  65. if cb.State() != StateClosed {
  66. t.Errorf("expected state to be Closed after success, got %v", cb.State())
  67. }
  68. }
  69. func TestRetry_Backoff(t *testing.T) {
  70. attempts := 0
  71. config := RetryConfig{
  72. MaxAttempts: 3,
  73. InitialBackoff: 10 * time.Millisecond,
  74. MaxBackoff: 100 * time.Millisecond,
  75. BackoffMultiplier: 2.0,
  76. Jitter: false, // Disable for predictable testing
  77. }
  78. ctx := context.Background()
  79. start := time.Now()
  80. err := WithRetry(ctx, config, func(ctx context.Context, attempt int) error {
  81. attempts++
  82. if attempt < 2 {
  83. return status.Error(codes.Unavailable, "unavailable")
  84. }
  85. return nil // Success on 3rd attempt
  86. })
  87. elapsed := time.Since(start)
  88. if err != nil {
  89. t.Fatalf("expected success after retries, got: %v", err)
  90. }
  91. if attempts != 3 {
  92. t.Errorf("expected 3 attempts, got %d", attempts)
  93. }
  94. // Expected: 10ms + 20ms = 30ms minimum
  95. if elapsed < 30*time.Millisecond {
  96. t.Errorf("expected at least 30ms elapsed, got %v", elapsed)
  97. }
  98. }
  99. func TestRetry_NonRetryableError(t *testing.T) {
  100. attempts := 0
  101. config := DefaultRetryConfig()
  102. ctx := context.Background()
  103. err := WithRetry(ctx, config, func(ctx context.Context, attempt int) error {
  104. attempts++
  105. return status.Error(codes.NotFound, "not found")
  106. })
  107. if err == nil {
  108. t.Fatal("expected error")
  109. }
  110. // Should not retry NotFound
  111. if attempts != 1 {
  112. t.Errorf("expected 1 attempt for non-retryable error, got %d", attempts)
  113. }
  114. }
  115. func TestRetry_ContextCancellation(t *testing.T) {
  116. config := DefaultRetryConfig()
  117. ctx, cancel := context.WithCancel(context.Background())
  118. // Cancel immediately
  119. cancel()
  120. err := WithRetry(ctx, config, func(ctx context.Context, attempt int) error {
  121. return status.Error(codes.Unavailable, "unavailable")
  122. })
  123. if !errors.Is(err, context.Canceled) {
  124. t.Errorf("expected context.Canceled, got: %v", err)
  125. }
  126. }
  127. func TestIsRetryable(t *testing.T) {
  128. tests := []struct {
  129. name string
  130. err error
  131. retryable bool
  132. }{
  133. {
  134. name: "Unavailable is retryable",
  135. err: status.Error(codes.Unavailable, "test"),
  136. retryable: true,
  137. },
  138. {
  139. name: "DeadlineExceeded is retryable",
  140. err: status.Error(codes.DeadlineExceeded, "test"),
  141. retryable: true,
  142. },
  143. {
  144. name: "NotFound is not retryable",
  145. err: status.Error(codes.NotFound, "test"),
  146. retryable: false,
  147. },
  148. {
  149. name: "PermissionDenied is not retryable",
  150. err: status.Error(codes.PermissionDenied, "test"),
  151. retryable: false,
  152. },
  153. {
  154. name: "Context canceled is not retryable",
  155. err: context.Canceled,
  156. retryable: false,
  157. },
  158. {
  159. name: "Wrapped context canceled is not retryable",
  160. err: fmt.Errorf("wrapped: %w", context.Canceled),
  161. retryable: false,
  162. },
  163. {
  164. name: "Wrapped context deadline exceeded is not retryable",
  165. err: fmt.Errorf("wrapped: %w", context.DeadlineExceeded),
  166. retryable: false,
  167. },
  168. }
  169. for _, tt := range tests {
  170. t.Run(tt.name, func(t *testing.T) {
  171. result := isRetryable(tt.err)
  172. if result != tt.retryable {
  173. t.Errorf("expected retryable=%v, got %v", tt.retryable, result)
  174. }
  175. })
  176. }
  177. }
  178. func TestCalculateBackoff(t *testing.T) {
  179. config := RetryConfig{
  180. InitialBackoff: 100 * time.Millisecond,
  181. MaxBackoff: 1 * time.Second,
  182. BackoffMultiplier: 2.0,
  183. Jitter: false,
  184. }
  185. tests := []struct {
  186. attempt int
  187. expected time.Duration
  188. }{
  189. {0, 100 * time.Millisecond},
  190. {1, 200 * time.Millisecond},
  191. {2, 400 * time.Millisecond},
  192. {3, 800 * time.Millisecond},
  193. {4, 1 * time.Second}, // Capped at MaxBackoff
  194. {5, 1 * time.Second}, // Still capped
  195. }
  196. for _, tt := range tests {
  197. result := calculateBackoff(tt.attempt, config)
  198. if result != tt.expected {
  199. t.Errorf("attempt %d: expected %v, got %v", tt.attempt, tt.expected, result)
  200. }
  201. }
  202. }