auth_test.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. /*
  2. Licensed under the Apache License, Version 2.0 (the "License");
  3. you may not use this file except in compliance with the License.
  4. You may obtain a copy of the License at
  5. http://www.apache.org/licenses/LICENSE-2.0
  6. Unless required by applicable law or agreed to in writing, software
  7. distributed under the License is distributed on an "AS IS" BASIS,
  8. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  9. See the License for the specific language governing permissions and
  10. limitations under the License.
  11. */
  12. package auth
  13. import (
  14. "context"
  15. "strings"
  16. "testing"
  17. "time"
  18. "github.com/aws/aws-sdk-go/aws"
  19. "github.com/aws/aws-sdk-go/aws/credentials"
  20. awssess "github.com/aws/aws-sdk-go/aws/session"
  21. "github.com/aws/aws-sdk-go/service/sts"
  22. "github.com/aws/aws-sdk-go/service/sts/stsiface"
  23. "github.com/stretchr/testify/assert"
  24. authv1 "k8s.io/api/authentication/v1"
  25. v1 "k8s.io/api/core/v1"
  26. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  27. clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
  28. esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1"
  29. esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
  30. fakesess "github.com/external-secrets/external-secrets/pkg/provider/aws/auth/fake"
  31. )
  32. const (
  33. esNamespaceKey = "es-namespace"
  34. platformTeamNsKey = "platform-team-ns"
  35. myServiceAccountKey = "my-service-account"
  36. otherNsName = "other-ns"
  37. )
  38. func TestNewSession(t *testing.T) {
  39. rows := []TestSessionRow{
  40. {
  41. name: "nil store",
  42. expectErr: "found nil store",
  43. store: nil,
  44. },
  45. {
  46. name: "not store spec",
  47. expectErr: "storeSpec is missing provider",
  48. store: &esv1beta1.SecretStore{},
  49. },
  50. {
  51. name: "store spec has no provider",
  52. expectErr: "storeSpec is missing provider",
  53. store: &esv1beta1.SecretStore{
  54. Spec: esv1beta1.SecretStoreSpec{},
  55. },
  56. },
  57. {
  58. name: "spec has no awssm field",
  59. expectErr: "Missing AWS field",
  60. store: &esv1beta1.SecretStore{
  61. Spec: esv1beta1.SecretStoreSpec{
  62. Provider: &esv1beta1.SecretStoreProvider{},
  63. },
  64. },
  65. },
  66. {
  67. name: "configure aws using environment variables",
  68. store: &esv1beta1.SecretStore{
  69. Spec: esv1beta1.SecretStoreSpec{
  70. Provider: &esv1beta1.SecretStoreProvider{
  71. AWS: &esv1beta1.AWSProvider{},
  72. },
  73. },
  74. },
  75. env: map[string]string{
  76. "AWS_ACCESS_KEY_ID": "1111",
  77. "AWS_SECRET_ACCESS_KEY": "2222",
  78. },
  79. expectProvider: true,
  80. expectedKeyID: "1111",
  81. expectedSecretKey: "2222",
  82. },
  83. {
  84. name: "configure aws using environment variables + assume role",
  85. stsProvider: func(*awssess.Session) stsiface.STSAPI {
  86. return &fakesess.AssumeRoler{
  87. AssumeRoleFunc: func(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
  88. assert.Equal(t, *input.RoleArn, "foo-bar-baz")
  89. return &sts.AssumeRoleOutput{
  90. AssumedRoleUser: &sts.AssumedRoleUser{
  91. Arn: aws.String("1123132"),
  92. AssumedRoleId: aws.String("xxxxx"),
  93. },
  94. Credentials: &sts.Credentials{
  95. AccessKeyId: aws.String("3333"),
  96. SecretAccessKey: aws.String("4444"),
  97. Expiration: aws.Time(time.Now().Add(time.Hour)),
  98. SessionToken: aws.String("6666"),
  99. },
  100. }, nil
  101. },
  102. }
  103. },
  104. store: &esv1beta1.SecretStore{
  105. Spec: esv1beta1.SecretStoreSpec{
  106. Provider: &esv1beta1.SecretStoreProvider{
  107. AWS: &esv1beta1.AWSProvider{
  108. Role: "foo-bar-baz",
  109. },
  110. },
  111. },
  112. },
  113. env: map[string]string{
  114. "AWS_ACCESS_KEY_ID": "1111",
  115. "AWS_SECRET_ACCESS_KEY": "2222",
  116. },
  117. expectProvider: true,
  118. expectedKeyID: "3333",
  119. expectedSecretKey: "4444",
  120. },
  121. {
  122. name: "error out when secret with credentials does not exist",
  123. namespace: "foo",
  124. store: &esv1beta1.SecretStore{
  125. Spec: esv1beta1.SecretStoreSpec{
  126. Provider: &esv1beta1.SecretStoreProvider{
  127. AWS: &esv1beta1.AWSProvider{
  128. Auth: esv1beta1.AWSAuth{
  129. SecretRef: &esv1beta1.AWSAuthSecretRef{
  130. AccessKeyID: esmeta.SecretKeySelector{
  131. Name: "othersecret",
  132. Key: "one",
  133. },
  134. SecretAccessKey: esmeta.SecretKeySelector{
  135. Name: "othersecret",
  136. Key: "two",
  137. },
  138. },
  139. },
  140. },
  141. },
  142. },
  143. },
  144. expectErr: `secrets "othersecret" not found`,
  145. },
  146. {
  147. name: "use credentials from secret to configure aws",
  148. namespace: "foo",
  149. store: &esv1beta1.SecretStore{
  150. Spec: esv1beta1.SecretStoreSpec{
  151. Provider: &esv1beta1.SecretStoreProvider{
  152. AWS: &esv1beta1.AWSProvider{
  153. Auth: esv1beta1.AWSAuth{
  154. SecretRef: &esv1beta1.AWSAuthSecretRef{
  155. AccessKeyID: esmeta.SecretKeySelector{
  156. Name: "onesecret",
  157. // Namespace is not set
  158. Key: "one",
  159. },
  160. SecretAccessKey: esmeta.SecretKeySelector{
  161. Name: "onesecret",
  162. // Namespace is not set
  163. Key: "two",
  164. },
  165. },
  166. },
  167. },
  168. },
  169. },
  170. },
  171. secrets: []v1.Secret{
  172. {
  173. ObjectMeta: metav1.ObjectMeta{
  174. Name: "onesecret",
  175. Namespace: "foo",
  176. },
  177. Data: map[string][]byte{
  178. "one": []byte("1111"),
  179. "two": []byte("2222"),
  180. },
  181. },
  182. },
  183. expectProvider: true,
  184. expectedKeyID: "1111",
  185. expectedSecretKey: "2222",
  186. },
  187. {
  188. name: "error out when secret key does not exist",
  189. namespace: "foo",
  190. store: &esv1beta1.SecretStore{
  191. Spec: esv1beta1.SecretStoreSpec{
  192. Provider: &esv1beta1.SecretStoreProvider{
  193. AWS: &esv1beta1.AWSProvider{
  194. Auth: esv1beta1.AWSAuth{
  195. SecretRef: &esv1beta1.AWSAuthSecretRef{
  196. AccessKeyID: esmeta.SecretKeySelector{
  197. Name: "brokensecret",
  198. Key: "one",
  199. },
  200. SecretAccessKey: esmeta.SecretKeySelector{
  201. Name: "brokensecret",
  202. Key: "two",
  203. },
  204. },
  205. },
  206. },
  207. },
  208. },
  209. },
  210. secrets: []v1.Secret{
  211. {
  212. ObjectMeta: metav1.ObjectMeta{
  213. Name: "brokensecret",
  214. Namespace: "foo",
  215. },
  216. Data: map[string][]byte{},
  217. },
  218. },
  219. expectErr: "missing SecretAccessKey",
  220. },
  221. {
  222. name: "should not be able to access secrets from different namespace",
  223. namespace: "foo",
  224. store: &esv1beta1.SecretStore{
  225. Spec: esv1beta1.SecretStoreSpec{
  226. Provider: &esv1beta1.SecretStoreProvider{
  227. AWS: &esv1beta1.AWSProvider{
  228. Auth: esv1beta1.AWSAuth{
  229. SecretRef: &esv1beta1.AWSAuthSecretRef{
  230. AccessKeyID: esmeta.SecretKeySelector{
  231. Name: "onesecret",
  232. Namespace: aws.String("evil"), // this should not be possible!
  233. Key: "one",
  234. },
  235. SecretAccessKey: esmeta.SecretKeySelector{
  236. Name: "onesecret",
  237. Namespace: aws.String("evil"),
  238. Key: "two",
  239. },
  240. },
  241. },
  242. },
  243. },
  244. },
  245. },
  246. secrets: []v1.Secret{
  247. {
  248. ObjectMeta: metav1.ObjectMeta{
  249. Name: "onesecret",
  250. Namespace: "evil",
  251. },
  252. Data: map[string][]byte{
  253. "one": []byte("1111"),
  254. "two": []byte("2222"),
  255. },
  256. },
  257. },
  258. expectErr: `secrets "onesecret" not found`,
  259. },
  260. {
  261. name: "ClusterStore should use credentials from a specific namespace",
  262. namespace: esNamespaceKey,
  263. store: &esv1beta1.ClusterSecretStore{
  264. TypeMeta: metav1.TypeMeta{
  265. APIVersion: esv1beta1.ClusterSecretStoreKindAPIVersion,
  266. Kind: esv1beta1.ClusterSecretStoreKind,
  267. },
  268. Spec: esv1beta1.SecretStoreSpec{
  269. Provider: &esv1beta1.SecretStoreProvider{
  270. AWS: &esv1beta1.AWSProvider{
  271. Auth: esv1beta1.AWSAuth{
  272. SecretRef: &esv1beta1.AWSAuthSecretRef{
  273. AccessKeyID: esmeta.SecretKeySelector{
  274. Name: "onesecret",
  275. Namespace: aws.String(platformTeamNsKey),
  276. Key: "one",
  277. },
  278. SecretAccessKey: esmeta.SecretKeySelector{
  279. Name: "onesecret",
  280. Namespace: aws.String(platformTeamNsKey),
  281. Key: "two",
  282. },
  283. },
  284. },
  285. },
  286. },
  287. },
  288. },
  289. secrets: []v1.Secret{
  290. {
  291. ObjectMeta: metav1.ObjectMeta{
  292. Name: "onesecret",
  293. Namespace: platformTeamNsKey,
  294. },
  295. Data: map[string][]byte{
  296. "one": []byte("1111"),
  297. "two": []byte("2222"),
  298. },
  299. },
  300. },
  301. expectProvider: true,
  302. expectedKeyID: "1111",
  303. expectedSecretKey: "2222",
  304. },
  305. {
  306. name: "ClusterStore should use credentials from a ExternalSecret namespace (referentAuth)",
  307. namespace: esNamespaceKey,
  308. store: &esv1beta1.ClusterSecretStore{
  309. TypeMeta: metav1.TypeMeta{
  310. APIVersion: esv1beta1.ClusterSecretStoreKindAPIVersion,
  311. Kind: esv1beta1.ClusterSecretStoreKind,
  312. },
  313. Spec: esv1beta1.SecretStoreSpec{
  314. Provider: &esv1beta1.SecretStoreProvider{
  315. AWS: &esv1beta1.AWSProvider{
  316. Auth: esv1beta1.AWSAuth{
  317. SecretRef: &esv1beta1.AWSAuthSecretRef{
  318. AccessKeyID: esmeta.SecretKeySelector{
  319. Name: "onesecret",
  320. Key: "one",
  321. },
  322. SecretAccessKey: esmeta.SecretKeySelector{
  323. Name: "onesecret",
  324. Key: "two",
  325. },
  326. },
  327. },
  328. },
  329. },
  330. },
  331. },
  332. secrets: []v1.Secret{
  333. {
  334. ObjectMeta: metav1.ObjectMeta{
  335. Name: "onesecret",
  336. Namespace: esNamespaceKey,
  337. },
  338. Data: map[string][]byte{
  339. "one": []byte("7777"),
  340. "two": []byte("4444"),
  341. },
  342. },
  343. },
  344. expectProvider: true,
  345. expectedKeyID: "7777",
  346. expectedSecretKey: "4444",
  347. },
  348. {
  349. name: "jwt auth via cluster secret store",
  350. namespace: esNamespaceKey,
  351. sa: &v1.ServiceAccount{
  352. ObjectMeta: metav1.ObjectMeta{
  353. Name: myServiceAccountKey,
  354. Namespace: otherNsName,
  355. Annotations: map[string]string{
  356. roleARNAnnotation: "my-sa-role",
  357. },
  358. },
  359. },
  360. jwtProvider: func(name, namespace, roleArn string, aud []string, region string) (credentials.Provider, error) {
  361. assert.Equal(t, myServiceAccountKey, name)
  362. assert.Equal(t, otherNsName, namespace)
  363. assert.Equal(t, "my-sa-role", roleArn)
  364. return fakesess.CredentialsProvider{
  365. RetrieveFunc: func() (credentials.Value, error) {
  366. return credentials.Value{
  367. AccessKeyID: "3333",
  368. SecretAccessKey: "4444",
  369. SessionToken: "1234",
  370. ProviderName: "fake",
  371. }, nil
  372. },
  373. IsExpiredFunc: func() bool { return false },
  374. }, nil
  375. },
  376. store: &esv1beta1.ClusterSecretStore{
  377. TypeMeta: metav1.TypeMeta{
  378. APIVersion: esv1beta1.ClusterSecretStoreKindAPIVersion,
  379. Kind: esv1beta1.ClusterSecretStoreKind,
  380. },
  381. Spec: esv1beta1.SecretStoreSpec{
  382. Provider: &esv1beta1.SecretStoreProvider{
  383. AWS: &esv1beta1.AWSProvider{
  384. Auth: esv1beta1.AWSAuth{
  385. JWTAuth: &esv1beta1.AWSJWTAuth{
  386. ServiceAccountRef: &esmeta.ServiceAccountSelector{
  387. Name: myServiceAccountKey,
  388. Namespace: aws.String(otherNsName),
  389. },
  390. },
  391. },
  392. },
  393. },
  394. },
  395. },
  396. expectProvider: true,
  397. expectedKeyID: "3333",
  398. expectedSecretKey: "4444",
  399. },
  400. {
  401. name: "configure aws using environment variables + assume role + check external id",
  402. stsProvider: func(*awssess.Session) stsiface.STSAPI {
  403. return &fakesess.AssumeRoler{
  404. AssumeRoleFunc: func(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
  405. assert.Equal(t, *input.ExternalId, "12345678")
  406. return &sts.AssumeRoleOutput{
  407. AssumedRoleUser: &sts.AssumedRoleUser{
  408. Arn: aws.String("1123132"),
  409. AssumedRoleId: aws.String("xxxxx"),
  410. },
  411. Credentials: &sts.Credentials{
  412. AccessKeyId: aws.String("3333"),
  413. SecretAccessKey: aws.String("4444"),
  414. Expiration: aws.Time(time.Now().Add(time.Hour)),
  415. SessionToken: aws.String("6666"),
  416. },
  417. }, nil
  418. },
  419. }
  420. },
  421. store: &esv1beta1.SecretStore{
  422. Spec: esv1beta1.SecretStoreSpec{
  423. Provider: &esv1beta1.SecretStoreProvider{
  424. AWS: &esv1beta1.AWSProvider{
  425. Role: "foo-bar-baz",
  426. ExternalID: "12345678",
  427. },
  428. },
  429. },
  430. },
  431. env: map[string]string{
  432. "AWS_ACCESS_KEY_ID": "1111",
  433. "AWS_SECRET_ACCESS_KEY": "2222",
  434. },
  435. expectProvider: true,
  436. expectedKeyID: "3333",
  437. expectedSecretKey: "4444",
  438. },
  439. }
  440. for i := range rows {
  441. row := rows[i]
  442. t.Run(row.name, func(t *testing.T) {
  443. testRow(t, row)
  444. })
  445. }
  446. }
  447. type TestSessionRow struct {
  448. name string
  449. store esv1beta1.GenericStore
  450. secrets []v1.Secret
  451. sa *v1.ServiceAccount
  452. jwtProvider jwtProviderFactory
  453. namespace string
  454. stsProvider STSProvider
  455. expectProvider bool
  456. expectErr string
  457. expectedKeyID string
  458. expectedSecretKey string
  459. env map[string]string
  460. }
  461. func testRow(t *testing.T, row TestSessionRow) {
  462. kc := clientfake.NewClientBuilder().Build()
  463. for i := range row.secrets {
  464. err := kc.Create(context.Background(), &row.secrets[i])
  465. assert.Nil(t, err)
  466. }
  467. for k, v := range row.env {
  468. t.Setenv(k, v)
  469. }
  470. if row.sa != nil {
  471. err := kc.Create(context.Background(), row.sa)
  472. assert.Nil(t, err)
  473. }
  474. err := kc.Create(context.Background(), &authv1.TokenRequest{
  475. ObjectMeta: metav1.ObjectMeta{
  476. Name: myServiceAccountKey,
  477. Namespace: otherNsName,
  478. },
  479. })
  480. assert.Nil(t, err)
  481. s, err := New(context.Background(), row.store, kc, row.namespace, row.stsProvider, row.jwtProvider)
  482. if !ErrorContains(err, row.expectErr) {
  483. t.Errorf("expected error %s but found %s", row.expectErr, err.Error())
  484. }
  485. // pass test on expected error
  486. if err != nil {
  487. return
  488. }
  489. if row.expectProvider && s == nil {
  490. t.Errorf("expected provider object, found nil")
  491. return
  492. }
  493. creds, _ := s.Config.Credentials.Get()
  494. assert.Equal(t, row.expectedKeyID, creds.AccessKeyID)
  495. assert.Equal(t, row.expectedSecretKey, creds.SecretAccessKey)
  496. }
  497. func TestSMEnvCredentials(t *testing.T) {
  498. k8sClient := clientfake.NewClientBuilder().Build()
  499. t.Setenv("AWS_SECRET_ACCESS_KEY", "1111")
  500. t.Setenv("AWS_ACCESS_KEY_ID", "2222")
  501. s, err := New(context.Background(), &esv1beta1.SecretStore{
  502. Spec: esv1beta1.SecretStoreSpec{
  503. Provider: &esv1beta1.SecretStoreProvider{
  504. // defaults
  505. AWS: &esv1beta1.AWSProvider{},
  506. },
  507. },
  508. }, k8sClient, "example-ns", DefaultSTSProvider, nil)
  509. assert.Nil(t, err)
  510. assert.NotNil(t, s)
  511. creds, err := s.Config.Credentials.Get()
  512. assert.Nil(t, err)
  513. assert.Equal(t, creds.AccessKeyID, "2222")
  514. assert.Equal(t, creds.SecretAccessKey, "1111")
  515. }
  516. func TestSMAssumeRole(t *testing.T) {
  517. k8sClient := clientfake.NewClientBuilder().Build()
  518. sts := &fakesess.AssumeRoler{
  519. AssumeRoleFunc: func(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error) {
  520. if *input.RoleArn == "chained-role-1" {
  521. return &sts.AssumeRoleOutput{
  522. AssumedRoleUser: &sts.AssumedRoleUser{
  523. Arn: aws.String("1111111"),
  524. AssumedRoleId: aws.String("yyyyy1"),
  525. },
  526. Credentials: &sts.Credentials{
  527. AccessKeyId: aws.String("77771"),
  528. SecretAccessKey: aws.String("88881"),
  529. Expiration: aws.Time(time.Now().Add(time.Hour)),
  530. SessionToken: aws.String("99991"),
  531. },
  532. }, nil
  533. } else if *input.RoleArn == "chained-role-2" {
  534. return &sts.AssumeRoleOutput{
  535. AssumedRoleUser: &sts.AssumedRoleUser{
  536. Arn: aws.String("2222222"),
  537. AssumedRoleId: aws.String("yyyyy2"),
  538. },
  539. Credentials: &sts.Credentials{
  540. AccessKeyId: aws.String("77772"),
  541. SecretAccessKey: aws.String("88882"),
  542. Expiration: aws.Time(time.Now().Add(time.Hour)),
  543. SessionToken: aws.String("99992"),
  544. },
  545. }, nil
  546. } else {
  547. // make sure the correct role is passed in
  548. assert.Equal(t, *input.RoleArn, "my-awesome-role")
  549. return &sts.AssumeRoleOutput{
  550. AssumedRoleUser: &sts.AssumedRoleUser{
  551. Arn: aws.String("1123132"),
  552. AssumedRoleId: aws.String("xxxxx"),
  553. },
  554. Credentials: &sts.Credentials{
  555. AccessKeyId: aws.String("3333"),
  556. SecretAccessKey: aws.String("4444"),
  557. Expiration: aws.Time(time.Now().Add(time.Hour)),
  558. SessionToken: aws.String("6666"),
  559. },
  560. }, nil
  561. }
  562. },
  563. }
  564. t.Setenv("AWS_SECRET_ACCESS_KEY", "1111")
  565. t.Setenv("AWS_ACCESS_KEY_ID", "2222")
  566. s, err := New(context.Background(), &esv1beta1.SecretStore{
  567. Spec: esv1beta1.SecretStoreSpec{
  568. Provider: &esv1beta1.SecretStoreProvider{
  569. // do assume role!
  570. AWS: &esv1beta1.AWSProvider{
  571. Role: "my-awesome-role",
  572. AdditionalRoles: []string{"chained-role-1", "chained-role-2"},
  573. },
  574. },
  575. },
  576. }, k8sClient, "example-ns", func(se *awssess.Session) stsiface.STSAPI {
  577. // check if the correct temporary credentials were used
  578. creds, err := se.Config.Credentials.Get()
  579. assert.Nil(t, err)
  580. if creds.SessionToken == "" {
  581. // called with credentials from envvars
  582. assert.Equal(t, creds.AccessKeyID, "2222")
  583. assert.Equal(t, creds.SecretAccessKey, "1111")
  584. } else if creds.SessionToken == "99991" {
  585. // called with chained role 1's credentials
  586. assert.Equal(t, creds.AccessKeyID, "77771")
  587. assert.Equal(t, creds.SecretAccessKey, "88881")
  588. } else {
  589. // called with chained role 2's credentials
  590. assert.Equal(t, creds.AccessKeyID, "77772")
  591. assert.Equal(t, creds.SecretAccessKey, "88882")
  592. }
  593. return sts
  594. }, nil)
  595. assert.Nil(t, err)
  596. assert.NotNil(t, s)
  597. creds, err := s.Config.Credentials.Get()
  598. assert.Nil(t, err)
  599. assert.Equal(t, creds.AccessKeyID, "3333")
  600. assert.Equal(t, creds.SecretAccessKey, "4444")
  601. }
  602. func ErrorContains(out error, want string) bool {
  603. if out == nil {
  604. return want == ""
  605. }
  606. if want == "" {
  607. return false
  608. }
  609. return strings.Contains(out.Error(), want)
  610. }