provider_test.go 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960
  1. /*
  2. Copyright © The ESO Authors
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. https://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package mysterybox
  14. import (
  15. "context"
  16. b64 "encoding/base64"
  17. "encoding/json"
  18. "errors"
  19. "fmt"
  20. "sync"
  21. "sync/atomic"
  22. "testing"
  23. "github.com/google/uuid"
  24. lru "github.com/hashicorp/golang-lru"
  25. "github.com/nebius/gosdk/auth"
  26. tassert "github.com/stretchr/testify/assert"
  27. corev1 "k8s.io/api/core/v1"
  28. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  29. ctrl "sigs.k8s.io/controller-runtime"
  30. k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
  31. clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
  32. esv1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1"
  33. esmeta "github.com/external-secrets/external-secrets/apis/meta/v1"
  34. "github.com/external-secrets/external-secrets/providers/v1/nebius/common/sdk/mysterybox"
  35. "github.com/external-secrets/external-secrets/providers/v1/nebius/common/sdk/mysterybox/fake"
  36. )
  37. const (
  38. tokenSecretName = "tokenSecretName"
  39. tokenSecretKey = "tokenSecretKey"
  40. saCredsSecretName = "saCredsSecretName"
  41. saCredsSecretKey = "saCredsSecretKey"
  42. authRefName = "authRefSecretName"
  43. authRefKey = "authRefSecretKey"
  44. apiDomain = "api.public"
  45. tokenToBeIssued = "token-to-be-issued"
  46. )
  47. var (
  48. logger = ctrl.Log.WithName("provider").WithName("nebius").WithName("mysterybox")
  49. )
  50. func setupClientWithTokenAuth(t *testing.T, entries []mysterybox.Entry, tokenGetter TokenGetter) (context.Context, *SecretsClient, *fake.Secret, k8sclient.Client, *fake.MysteryboxService) {
  51. t.Helper()
  52. ctx := context.Background()
  53. namespace := uuid.NewString()
  54. mysteryboxService := fake.InitMysteryboxService()
  55. k8sClient := clientfake.NewClientBuilder().Build()
  56. secret := mysteryboxService.CreateSecret(entries)
  57. provider := newProvider(
  58. t,
  59. func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  60. return fake.NewFakeMysteryboxClient(mysteryboxService), nil
  61. },
  62. tokenGetter,
  63. )
  64. createK8sSecret(ctx, t, k8sClient, namespace, tokenSecretName, tokenSecretKey, []byte("token"))
  65. store := newNebiusMysteryboxSecretStoreWithAuthTokenKey(apiDomain, namespace, tokenSecretName, tokenSecretKey)
  66. client, err := provider.NewClient(ctx, store, k8sClient, namespace)
  67. tassert.NoError(t, err)
  68. mysteryboxSecretsClient, ok := client.(*SecretsClient)
  69. tassert.True(t, ok, "expected *SecretsClient, got %T", client)
  70. return ctx, mysteryboxSecretsClient, secret, k8sClient, mysteryboxService
  71. }
  72. func TestNewClient_GetTokenError(t *testing.T) {
  73. t.Parallel()
  74. tokenGetter := faketokenGetter{returnError: true}
  75. ctx := context.Background()
  76. namespace := uuid.NewString()
  77. mysteryboxService := fake.InitMysteryboxService()
  78. k8sClient := clientfake.NewClientBuilder().Build()
  79. createK8sSecret(ctx, t, k8sClient, namespace, saCredsSecretName, saCredsSecretKey, []byte("PRIVATE KEY"))
  80. provider := newProvider(
  81. t,
  82. func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  83. return fake.NewFakeMysteryboxClient(mysteryboxService), nil
  84. },
  85. &tokenGetter,
  86. )
  87. _, err := provider.NewClient(ctx, newNebiusMysteryboxSecretStoreWithServiceAccountCreds(apiDomain, namespace, saCredsSecretName, saCredsSecretKey), k8sClient, namespace)
  88. tassert.Error(t, err)
  89. tassert.ErrorContains(t, err, "failed to retrieve iam token by credentials")
  90. }
  91. func TestGetSecret(t *testing.T) {
  92. t.Parallel()
  93. entries := []mysterybox.Entry{
  94. {Key: "key1", StringValue: "string"},
  95. {Key: "key2", StringValue: "string2"},
  96. {Key: "key3", BinaryValue: []byte("binaryValue")},
  97. }
  98. tests := []struct {
  99. name string
  100. prepare func(ctx context.Context, client *SecretsClient, secret *fake.Secret, svc *fake.MysteryboxService) ([]byte, error)
  101. expectJSON map[string]string
  102. expectRaw []byte
  103. }{
  104. {
  105. name: "get all entries as JSON",
  106. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret, _ *fake.MysteryboxService) ([]byte, error) {
  107. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id})
  108. },
  109. expectJSON: map[string]string{
  110. "key1": "string",
  111. "key2": "string2",
  112. "key3": b64.StdEncoding.EncodeToString([]byte("binaryValue")),
  113. },
  114. },
  115. {
  116. name: "string entry by key",
  117. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret, _ *fake.MysteryboxService) ([]byte, error) {
  118. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: "key1"})
  119. },
  120. expectRaw: []byte("string"),
  121. },
  122. {
  123. name: "binary entry by key",
  124. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret, _ *fake.MysteryboxService) ([]byte, error) {
  125. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: "key3"})
  126. },
  127. expectRaw: []byte("binaryValue"),
  128. },
  129. }
  130. for _, tt := range tests {
  131. t.Run(tt.name, func(t *testing.T) {
  132. t.Parallel()
  133. ctx, client, secret, _, svc := setupClientWithTokenAuth(t, entries, nil)
  134. result, err := tt.prepare(ctx, client, secret, svc)
  135. tassert.NoError(t, err)
  136. if tt.expectJSON != nil {
  137. tassert.Equal(t, tt.expectJSON, unmarshalStringMap(t, result))
  138. } else {
  139. tassert.Equal(t, tt.expectRaw, result)
  140. }
  141. })
  142. }
  143. }
  144. func TestGetSecret_ByVersionId(t *testing.T) {
  145. t.Parallel()
  146. ctx, client, secret, _, mboxService := setupClientWithTokenAuth(t, []mysterybox.Entry{{Key: "key", StringValue: "string_value"}}, nil)
  147. _, err := mboxService.CreateNewSecretVersion(secret.Id, []mysterybox.Entry{
  148. {Key: "new_key", StringValue: "updated_string_value"},
  149. {Key: "new", StringValue: "new"},
  150. })
  151. tassert.NoError(t, err)
  152. result, err := client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: "key", Version: secret.VersionId})
  153. tassert.NoError(t, err)
  154. tassert.Equal(t, []byte("string_value"), result)
  155. }
  156. func TestGetSecretMap(t *testing.T) {
  157. t.Parallel()
  158. allEntries := []mysterybox.Entry{
  159. {Key: "key1", StringValue: "string"},
  160. {Key: "key2", StringValue: "string2"},
  161. {Key: "key3", BinaryValue: []byte("binaryValue")},
  162. }
  163. tests := []struct {
  164. name string
  165. entries []mysterybox.Entry
  166. prepare func(ctx context.Context, client *SecretsClient, secret *fake.Secret, svc *fake.MysteryboxService) (map[string][]byte, error)
  167. expectMap map[string][]byte
  168. }{
  169. {
  170. name: "all entries",
  171. entries: allEntries,
  172. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret, _ *fake.MysteryboxService) (map[string][]byte, error) {
  173. return client.GetSecretMap(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id})
  174. },
  175. expectMap: map[string][]byte{
  176. "key1": []byte("string"),
  177. "key2": []byte("string2"),
  178. "key3": []byte("binaryValue"),
  179. },
  180. },
  181. {
  182. name: "by version id",
  183. entries: []mysterybox.Entry{{Key: "key", StringValue: "string_value"}},
  184. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret, svc *fake.MysteryboxService) (map[string][]byte, error) {
  185. _, err := svc.CreateNewSecretVersion(secret.Id, []mysterybox.Entry{{Key: "new_key", StringValue: "updated_string_value"}, {Key: "new", StringValue: "new"}})
  186. if err != nil {
  187. return nil, err
  188. }
  189. return client.GetSecretMap(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Version: secret.VersionId})
  190. },
  191. expectMap: map[string][]byte{
  192. "key": []byte("string_value"),
  193. },
  194. },
  195. }
  196. for _, tt := range tests {
  197. t.Run(tt.name, func(t *testing.T) {
  198. t.Parallel()
  199. ctx, client, secret, _, svc := setupClientWithTokenAuth(t, tt.entries, nil)
  200. result, err := tt.prepare(ctx, client, secret, svc)
  201. tassert.NoError(t, err)
  202. tassert.Equal(t, tt.expectMap, result)
  203. })
  204. }
  205. }
  206. func TestNewClient_ValidationErrors(t *testing.T) {
  207. t.Parallel()
  208. ctx := context.Background()
  209. namespace := uuid.NewString()
  210. mysteryboxService := fake.InitMysteryboxService()
  211. k8sClient := clientfake.NewClientBuilder().Build()
  212. createK8sSecret(ctx, t, k8sClient, namespace, tokenSecretName, tokenSecretKey, []byte("token"))
  213. tokenToIssue := tokenToBeIssued
  214. notExistingSecretName := "not-existing-secret"
  215. notExistingSecretKey := "not-existing-secret-key"
  216. cache, err := lru.New(10)
  217. tassert.NoError(t, err)
  218. tokenGetter := &faketokenGetter{tokenToIssue: tokenToIssue}
  219. newProvider := func() *Provider {
  220. return &Provider{
  221. Logger: logger,
  222. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  223. return fake.NewFakeMysteryboxClient(mysteryboxService), nil
  224. },
  225. TokenGetter: tokenGetter,
  226. mysteryboxClientsCache: cache,
  227. }
  228. }
  229. tests := []struct {
  230. name string
  231. storeSpec func() *esv1.SecretStore
  232. expectErr string
  233. }{
  234. {
  235. name: "missing api domain",
  236. storeSpec: func() *esv1.SecretStore {
  237. return &esv1.SecretStore{
  238. ObjectMeta: metav1.ObjectMeta{Namespace: namespace},
  239. Spec: esv1.SecretStoreSpec{
  240. Provider: &esv1.SecretStoreProvider{
  241. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{},
  242. },
  243. },
  244. }
  245. },
  246. expectErr: errMissingAPIDomain,
  247. },
  248. {
  249. name: "missing auth options",
  250. storeSpec: func() *esv1.SecretStore {
  251. return &esv1.SecretStore{
  252. ObjectMeta: metav1.ObjectMeta{Namespace: namespace},
  253. Spec: esv1.SecretStoreSpec{
  254. Provider: &esv1.SecretStoreProvider{
  255. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{APIDomain: apiDomain},
  256. },
  257. },
  258. }
  259. },
  260. expectErr: errMissingAuthOptions,
  261. },
  262. {
  263. name: "specified token secret does not exist in kubernetes secrets",
  264. storeSpec: func() *esv1.SecretStore {
  265. return &esv1.SecretStore{
  266. ObjectMeta: metav1.ObjectMeta{Namespace: namespace},
  267. Spec: esv1.SecretStoreSpec{
  268. Provider: &esv1.SecretStoreProvider{
  269. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{
  270. APIDomain: apiDomain,
  271. Auth: esv1.NebiusAuth{
  272. Token: esmeta.SecretKeySelector{
  273. Namespace: &namespace,
  274. Name: notExistingSecretName,
  275. Key: tokenSecretKey,
  276. },
  277. },
  278. },
  279. },
  280. },
  281. }
  282. },
  283. expectErr: fmt.Sprintf("read token secret %s/%s: cannot get Kubernetes secret", namespace, notExistingSecretName),
  284. },
  285. {
  286. name: "specified service account " +
  287. "creds secret does not exist in kubernetes secrets",
  288. storeSpec: func() *esv1.SecretStore {
  289. return &esv1.SecretStore{
  290. ObjectMeta: metav1.ObjectMeta{Namespace: namespace},
  291. Spec: esv1.SecretStoreSpec{
  292. Provider: &esv1.SecretStoreProvider{
  293. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{
  294. APIDomain: apiDomain,
  295. Auth: esv1.NebiusAuth{
  296. ServiceAccountCreds: esmeta.SecretKeySelector{
  297. Namespace: &namespace,
  298. Name: notExistingSecretName,
  299. Key: "secretKey",
  300. },
  301. },
  302. },
  303. },
  304. },
  305. }
  306. },
  307. expectErr: fmt.Sprintf("read service account creds %s/%s: cannot get Kubernetes secret", namespace, notExistingSecretName),
  308. },
  309. {
  310. name: "specified token secret's key does not exist in the secret",
  311. storeSpec: func() *esv1.SecretStore {
  312. return &esv1.SecretStore{
  313. ObjectMeta: metav1.ObjectMeta{Namespace: namespace},
  314. Spec: esv1.SecretStoreSpec{
  315. Provider: &esv1.SecretStoreProvider{
  316. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{
  317. APIDomain: apiDomain,
  318. Auth: esv1.NebiusAuth{
  319. Token: esmeta.SecretKeySelector{
  320. Namespace: &namespace,
  321. Name: tokenSecretName,
  322. Key: notExistingSecretKey,
  323. },
  324. },
  325. },
  326. },
  327. },
  328. }
  329. },
  330. expectErr: fmt.Sprintf("cannot find secret data for key: %q", notExistingSecretKey),
  331. },
  332. }
  333. for _, tt := range tests {
  334. t.Run(tt.name, func(t *testing.T) {
  335. t.Parallel()
  336. p := newProvider()
  337. store := tt.storeSpec()
  338. _, err := p.NewClient(ctx, store, k8sClient, namespace)
  339. tassert.Error(t, err)
  340. tassert.ErrorContains(t, err, tt.expectErr)
  341. })
  342. }
  343. }
  344. func TestNewClient_AuthWithSecretAccountCreds(t *testing.T) {
  345. t.Parallel()
  346. ctx := context.Background()
  347. namespace := uuid.NewString()
  348. mysteryboxService := fake.InitMysteryboxService()
  349. k8sClient := clientfake.NewClientBuilder().Build()
  350. secret := mysteryboxService.CreateSecret([]mysterybox.Entry{{Key: "k", StringValue: "v"}})
  351. providedCreds, _ := json.Marshal(&auth.ServiceAccountCredentials{
  352. SubjectCredentials: auth.SubjectCredentials{
  353. PrivateKey: "-----BEGIN PRIVATE KEY-----\nTEST-KEY\n-----END PRIVATE KEY-----",
  354. KeyID: "keyId",
  355. Subject: "subjectId",
  356. Issuer: "subjectId",
  357. },
  358. })
  359. tokenToIssue := tokenToBeIssued
  360. tokenGetter := &faketokenGetter{
  361. tokenToIssue: tokenToIssue,
  362. }
  363. cache, err := lru.New(10)
  364. tassert.NoError(t, err)
  365. p := &Provider{
  366. Logger: logger,
  367. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  368. return fake.NewFakeMysteryboxClient(mysteryboxService), nil
  369. },
  370. mysteryboxClientsCache: cache,
  371. }
  372. settokenGetterWorkaround(tokenGetter, p)
  373. createK8sSecret(ctx, t, k8sClient, namespace, authRefName, authRefKey, providedCreds)
  374. store := newNebiusMysteryboxSecretStoreWithServiceAccountCreds(apiDomain, namespace, authRefName, authRefKey)
  375. client, err := p.NewClient(ctx, store, k8sClient, namespace)
  376. tassert.NoError(t, err)
  377. msc, ok := client.(*SecretsClient)
  378. tassert.True(t, ok, "expected *MysteryboxSecretsClient, got %T", client)
  379. tassert.Equal(t, tokenToIssue, msc.token, fmt.Sprintf("token mismatch: got %q want %q (issued by TokenGetter)", msc.token, tokenToIssue))
  380. // also ensure TokenGetter was exercised with the domain and creds we expect
  381. tassert.Equal(t, int32(1), tokenGetter.calls, "expected TokenGetter to be called once")
  382. tassert.Equal(t, apiDomain, tokenGetter.gotDomain, "expected TokenGetter to be called with the correct domain")
  383. tassert.Equal(t, string(providedCreds), tokenGetter.gotCreds, "expected TokenGetter to be called with the correct creds")
  384. tassert.Nil(t, tokenGetter.gotCACert, "expected TokenGetter to be called without CA cert")
  385. got, err := msc.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: "k"})
  386. tassert.NoError(t, err)
  387. tassert.Equal(t, []byte("v"), got)
  388. }
  389. func TestGetSecret_NotFound(t *testing.T) {
  390. t.Parallel()
  391. // Use table-driven tests to cover all NotFound scenarios in one place
  392. cases := []struct {
  393. name string
  394. entries []mysterybox.Entry
  395. prepare func(ctx context.Context, client *SecretsClient, secret *fake.Secret) ([]byte, error)
  396. expectErrEqual func(t *testing.T, err error, secret *fake.Secret)
  397. }{
  398. {
  399. name: "SecretID not found",
  400. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  401. prepare: func(ctx context.Context, client *SecretsClient, _ *fake.Secret) ([]byte, error) {
  402. desiredSecretId := "notexists"
  403. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: desiredSecretId})
  404. },
  405. expectErrEqual: func(t *testing.T, err error, _ *fake.Secret) {
  406. tassert.Error(t, err)
  407. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  408. tassert.EqualError(t, err, fmt.Errorf(errSecretNotFound, "notexists", esv1.NoSecretErr).Error())
  409. },
  410. },
  411. {
  412. name: "Version not found",
  413. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  414. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret) ([]byte, error) {
  415. desiredVersion := "notexistversion"
  416. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Version: desiredVersion})
  417. },
  418. expectErrEqual: func(t *testing.T, err error, secret *fake.Secret) {
  419. tassert.Error(t, err)
  420. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  421. tassert.EqualError(t, err, fmt.Errorf(errSecretVersionNotFound, "notexistversion", secret.Id, esv1.NoSecretErr).Error())
  422. },
  423. },
  424. {
  425. name: "Property key not found in version",
  426. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  427. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret) ([]byte, error) {
  428. desiredKey := "key"
  429. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Version: secret.VersionId, Property: desiredKey})
  430. },
  431. expectErrEqual: func(t *testing.T, err error, secret *fake.Secret) {
  432. tassert.Error(t, err)
  433. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  434. tassert.EqualError(t, err, fmt.Errorf(errSecretVersionByKeyNotFound, secret.VersionId, secret.Id, "key", esv1.NoSecretErr).Error())
  435. },
  436. },
  437. {
  438. name: "Property key not found",
  439. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  440. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret) ([]byte, error) {
  441. desiredKey := "key"
  442. return client.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: desiredKey})
  443. },
  444. expectErrEqual: func(t *testing.T, err error, secret *fake.Secret) {
  445. tassert.Error(t, err)
  446. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  447. tassert.EqualError(t, err, fmt.Errorf(errSecretByKeyNotFound, "key", secret.Id, esv1.NoSecretErr).Error())
  448. },
  449. },
  450. }
  451. for _, tc := range cases {
  452. t.Run(tc.name, func(t *testing.T) {
  453. t.Parallel()
  454. ctx, client, secret, _, _ := setupClientWithTokenAuth(t, tc.entries, nil)
  455. _, err := tc.prepare(ctx, client, secret)
  456. tc.expectErrEqual(t, err, secret)
  457. })
  458. }
  459. }
  460. func TestGetSecretMap_NotFound(t *testing.T) {
  461. t.Parallel()
  462. cases := []struct {
  463. name string
  464. entries []mysterybox.Entry
  465. prepare func(ctx context.Context, client *SecretsClient, secret *fake.Secret) (map[string][]byte, error)
  466. expectErrEqual func(t *testing.T, err error, secret *fake.Secret)
  467. }{
  468. {
  469. name: "SecretID not found",
  470. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  471. prepare: func(ctx context.Context, client *SecretsClient, _ *fake.Secret) (map[string][]byte, error) {
  472. desiredSecretId := "notexists"
  473. return client.GetSecretMap(ctx, esv1.ExternalSecretDataRemoteRef{Key: desiredSecretId})
  474. },
  475. expectErrEqual: func(t *testing.T, err error, _ *fake.Secret) {
  476. tassert.Error(t, err)
  477. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  478. tassert.EqualError(t, err, fmt.Errorf(errSecretNotFound, "notexists", esv1.NoSecretErr).Error())
  479. },
  480. },
  481. {
  482. name: "Version not found",
  483. entries: []mysterybox.Entry{{Key: "key1", StringValue: "string"}},
  484. prepare: func(ctx context.Context, client *SecretsClient, secret *fake.Secret) (map[string][]byte, error) {
  485. desiredVersion := "notexistversion"
  486. return client.GetSecretMap(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Version: desiredVersion})
  487. },
  488. expectErrEqual: func(t *testing.T, err error, secret *fake.Secret) {
  489. tassert.Error(t, err)
  490. tassert.ErrorIs(t, err, esv1.NoSecretErr)
  491. tassert.EqualError(t, err, fmt.Errorf(errSecretVersionNotFound, "notexistversion", secret.Id, esv1.NoSecretErr).Error())
  492. },
  493. },
  494. }
  495. for _, tc := range cases {
  496. t.Run(tc.name, func(t *testing.T) {
  497. t.Parallel()
  498. ctx, client, secret, _, _ := setupClientWithTokenAuth(t, tc.entries, nil)
  499. _, err := tc.prepare(ctx, client, secret)
  500. tc.expectErrEqual(t, err, secret)
  501. })
  502. }
  503. }
  504. func TestCreateOrGetMysteryboxClient_CachesByKey(t *testing.T) {
  505. t.Parallel()
  506. ctx := context.Background()
  507. cache, err := lru.New(10)
  508. tassert.NoError(t, err)
  509. var factoryCalls int32
  510. p := &Provider{
  511. Logger: logger,
  512. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  513. atomic.AddInt32(&factoryCalls, 1)
  514. return fake.NewFakeMysteryboxClient(nil), nil
  515. },
  516. mysteryboxClientsCache: cache,
  517. }
  518. // same domain + same CA
  519. _, err = p.createOrGetMysteryboxClient(ctx, "api.nebius.example", []byte("CA1"))
  520. tassert.NoError(t, err)
  521. _, err = p.createOrGetMysteryboxClient(ctx, "api.nebius.example", []byte("CA1"))
  522. tassert.NoError(t, err)
  523. // different CA
  524. _, err = p.createOrGetMysteryboxClient(ctx, "api.nebius.example", []byte("CA2"))
  525. tassert.NoError(t, err)
  526. _, err = p.createOrGetMysteryboxClient(ctx, "other.nebius.example", []byte("CA1"))
  527. tassert.NoError(t, err)
  528. tassert.Equal(t, int32(3), atomic.LoadInt32(&factoryCalls), fmt.Sprintf("factory called %d times, want %d", atomic.LoadInt32(&factoryCalls), 3))
  529. }
  530. func TestCreateOrGetMysteryboxClient_EmptyCA_EqualsNil(t *testing.T) {
  531. t.Parallel()
  532. ctx := context.Background()
  533. cache, err := lru.New(10)
  534. tassert.NoError(t, err)
  535. var factoryCalls int32
  536. p := &Provider{
  537. Logger: logger,
  538. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  539. atomic.AddInt32(&factoryCalls, 1)
  540. return fake.NewFakeMysteryboxClient(nil), nil
  541. },
  542. mysteryboxClientsCache: cache,
  543. }
  544. _, err = p.createOrGetMysteryboxClient(ctx, "api.nebius.example", nil)
  545. tassert.NoError(t, err)
  546. _, err = p.createOrGetMysteryboxClient(ctx, "api.nebius.example", []byte{})
  547. tassert.NoError(t, err)
  548. tassert.Equal(t, int32(1), atomic.LoadInt32(&factoryCalls), fmt.Sprintf("factory called %d times, want %d when CA=nil and CA=empty should map to same key", factoryCalls, 1))
  549. }
  550. func TestMysteryboxClientsCache_EvictionClosesClient(t *testing.T) {
  551. t.Parallel()
  552. ctx := context.Background()
  553. var created []*fake.FakeMysteryboxClient
  554. p := &Provider{
  555. Logger: logger,
  556. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  557. c := &fake.FakeMysteryboxClient{}
  558. created = append(created, c)
  559. return c, nil
  560. },
  561. }
  562. setCacheSizeWorkaround(t, 1, p)
  563. _, err := p.createOrGetMysteryboxClient(ctx, "domain-a", nil)
  564. tassert.NoError(t, err)
  565. tassert.Len(t, created, 1, "expected 1 client created, got %d", len(created))
  566. _, err = p.createOrGetMysteryboxClient(ctx, "domain-b", nil)
  567. tassert.NoError(t, err)
  568. tassert.Len(t, created, 2, "expected 2 clients created, got %d", len(created))
  569. // clients are not closed because of the eviction policy for provider's mysterybox clients connections cache
  570. tassert.Equal(t, int32(0), atomic.LoadInt32(&created[0].Closed), "expected second client not to be closed")
  571. tassert.Equal(t, int32(0), atomic.LoadInt32(&created[1].Closed), "expected second client not to be closed")
  572. }
  573. // concurrent tests
  574. func TestCreateOrGetMysteryboxClient_Concurrent_SingleClient(t *testing.T) {
  575. t.Parallel()
  576. clientData := ClientData{domain: "api.nebius.example", ca: []byte("CA1")}
  577. ctx := context.Background()
  578. cache, err := lru.New(10)
  579. tassert.NoError(t, err)
  580. var factoryCalls int32
  581. p := &Provider{
  582. Logger: logger,
  583. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  584. atomic.AddInt32(&factoryCalls, 1)
  585. return fake.NewFakeMysteryboxClient(nil), nil
  586. },
  587. mysteryboxClientsCache: cache,
  588. }
  589. const goroutines = 8
  590. var wg sync.WaitGroup
  591. wg.Add(goroutines)
  592. start := make(chan struct{})
  593. errs := make([]error, goroutines)
  594. for i := range goroutines {
  595. go func(ix int) {
  596. defer wg.Done()
  597. <-start
  598. _, err := p.createOrGetMysteryboxClient(ctx, clientData.domain, clientData.ca)
  599. errs[ix] = err
  600. }(i)
  601. }
  602. close(start)
  603. wg.Wait()
  604. for i, err := range errs {
  605. if err != nil {
  606. tassert.NoError(t, err, "goroutine %d", i)
  607. }
  608. }
  609. tassert.Equal(t, int32(1), atomic.LoadInt32(&factoryCalls), fmt.Sprintf("factory called %d times, want %d for concurrent same-key requests", factoryCalls, 1))
  610. }
  611. func TestCreateOrGetMysteryboxClient_Concurrent_MultipleClients(t *testing.T) {
  612. clientRequests := []ClientData{
  613. {domain: "api.nebius.example1", ca: []byte("CA1")},
  614. {domain: "api.nebius.example1", ca: []byte("CA1")}, // duplicate
  615. {domain: "api.nebius.example1", ca: []byte("CA2")}, // the same domain, different CA
  616. {domain: "api.nebius.example2", ca: []byte("CA2")}, // different domain, the same CA
  617. {domain: "api.nebius.example1", ca: []byte{}}, // the same domain, empty CA
  618. }
  619. ctx := context.Background()
  620. cache, err := lru.New(10)
  621. tassert.NoError(t, err)
  622. var factoryCalls int32
  623. p := &Provider{
  624. Logger: logger,
  625. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  626. atomic.AddInt32(&factoryCalls, 1)
  627. return fake.NewFakeMysteryboxClient(nil), nil
  628. },
  629. mysteryboxClientsCache: cache,
  630. }
  631. var wg sync.WaitGroup
  632. wg.Add(len(clientRequests))
  633. start := make(chan struct{})
  634. errs := make([]error, len(clientRequests))
  635. for i, r := range clientRequests {
  636. go func(ix int, req ClientData) {
  637. defer wg.Done()
  638. <-start
  639. _, err := p.createOrGetMysteryboxClient(ctx, req.domain, req.ca)
  640. errs[ix] = err
  641. }(i, r)
  642. }
  643. close(start)
  644. wg.Wait()
  645. for i, err := range errs {
  646. if err != nil {
  647. tassert.NoError(t, err, "goroutine %d", i)
  648. }
  649. }
  650. tassert.Equal(t, int32(4), atomic.LoadInt32(&factoryCalls), fmt.Sprintf("factory called %d times, want %d", atomic.LoadInt32(&factoryCalls), 4))
  651. }
  652. func TestMysteryboxClientsCache_ConcurrentEviction_CloseOnce(t *testing.T) {
  653. ctx := context.Background()
  654. var created []*fake.FakeMysteryboxClient
  655. var mu sync.Mutex
  656. p := &Provider{
  657. Logger: logger,
  658. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  659. c := &fake.FakeMysteryboxClient{}
  660. mu.Lock()
  661. created = append(created, c)
  662. mu.Unlock()
  663. return c, nil
  664. },
  665. }
  666. setCacheSizeWorkaround(t, 1, p)
  667. var wg sync.WaitGroup
  668. wg.Add(3)
  669. start := make(chan struct{})
  670. go func() {
  671. defer wg.Done()
  672. <-start
  673. _, _ = p.createOrGetMysteryboxClient(ctx, "domain-a", nil)
  674. }()
  675. go func() {
  676. defer wg.Done()
  677. <-start
  678. _, _ = p.createOrGetMysteryboxClient(ctx, "domain-b", nil)
  679. }()
  680. go func() {
  681. defer wg.Done()
  682. <-start
  683. _, _ = p.createOrGetMysteryboxClient(ctx, "domain-c", nil)
  684. }()
  685. close(start)
  686. wg.Wait()
  687. tassert.Len(t, created, 3, "expected 3 clients created, got %d", len(created))
  688. // clients are not closed because of the eviction policy for provider's mysterybox clients connections cache
  689. tassert.Equal(t, int32(0), atomic.LoadInt32(&created[0].Closed), "expected first client not to be closed on eviction")
  690. tassert.Equal(t, int32(0), atomic.LoadInt32(&created[1].Closed), "expected first client not to be closed on eviction")
  691. tassert.Equal(t, int32(0), atomic.LoadInt32(&created[2].Closed), "expected first client not to be closed on eviction")
  692. }
  693. func TestNewClient_Concurrent_SameConfig_SingleClient_DifferentTokens(t *testing.T) {
  694. ctx := context.Background()
  695. namespace := uuid.NewString()
  696. mboxSvc := fake.InitMysteryboxService()
  697. k8sClient := clientfake.NewClientBuilder().Build()
  698. secret := mboxSvc.CreateSecret([]mysterybox.Entry{{Key: "k", StringValue: "v"}})
  699. var factoryCalls int32
  700. tokenToIssue := tokenToBeIssued
  701. tokenGetter := &faketokenGetter{tokenToIssue: tokenToIssue}
  702. p := &Provider{
  703. Logger: logger,
  704. NewMysteryboxClient: func(ctx context.Context, apiDomain string, caCertificate []byte) (mysterybox.Client, error) {
  705. atomic.AddInt32(&factoryCalls, 1)
  706. return fake.NewFakeMysteryboxClient(mboxSvc), nil
  707. },
  708. }
  709. settokenGetterWorkaround(tokenGetter, p)
  710. creds := []byte(`{"private_key":"KEY","key_id":"id","subject":"sub","issuer":"iss"}`)
  711. createK8sSecret(ctx, t, k8sClient, namespace, authRefName, authRefKey, creds)
  712. store := newNebiusMysteryboxSecretStoreWithServiceAccountCreds(apiDomain, namespace, authRefName, authRefKey)
  713. const goroutines = 10
  714. var wg sync.WaitGroup
  715. wg.Add(goroutines)
  716. start := make(chan struct{})
  717. clients := make([]esv1.SecretsClient, goroutines)
  718. errs := make([]error, goroutines)
  719. for i := range goroutines {
  720. go func(ix int) {
  721. defer wg.Done()
  722. <-start
  723. c, err := p.NewClient(ctx, store, k8sClient, namespace)
  724. clients[ix], errs[ix] = c, err
  725. }(i)
  726. }
  727. close(start)
  728. wg.Wait()
  729. for i := range goroutines {
  730. tassert.NoError(t, errs[i], "NewClient error: %w", errs[i])
  731. msc := clients[i].(*SecretsClient)
  732. got, err := msc.GetSecret(ctx, esv1.ExternalSecretDataRemoteRef{Key: secret.Id, Property: "k"})
  733. tassert.NoError(t, err)
  734. tassert.Equal(t, []byte("v"), got)
  735. }
  736. tassert.Equal(t, int32(goroutines), atomic.LoadInt32(&tokenGetter.calls), fmt.Sprintf("TokenGetter.GetToken called %d times, want %d", factoryCalls, goroutines))
  737. tassert.Equal(t, int32(1), atomic.LoadInt32(&factoryCalls), fmt.Sprintf("NewMysteryboxClient called %d times, want 1", factoryCalls))
  738. }
  739. // helpers
  740. func newNebiusMysteryboxSecretStoreWithAuthTokenKey(apiDomain, namespace, tokenSecretName, tokenSecretKey string) esv1.GenericStore {
  741. return &esv1.SecretStore{
  742. ObjectMeta: metav1.ObjectMeta{
  743. Namespace: namespace,
  744. },
  745. Spec: esv1.SecretStoreSpec{
  746. Provider: &esv1.SecretStoreProvider{
  747. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{
  748. APIDomain: apiDomain,
  749. Auth: esv1.NebiusAuth{
  750. Token: esmeta.SecretKeySelector{
  751. Name: tokenSecretName,
  752. Key: tokenSecretKey,
  753. },
  754. },
  755. },
  756. },
  757. },
  758. }
  759. }
  760. func newNebiusMysteryboxSecretStoreWithServiceAccountCreds(apiDomain, namespace, keySecretName, keySecretKey string) esv1.GenericStore {
  761. return &esv1.SecretStore{
  762. ObjectMeta: metav1.ObjectMeta{
  763. Namespace: namespace,
  764. },
  765. Spec: esv1.SecretStoreSpec{
  766. Provider: &esv1.SecretStoreProvider{
  767. NebiusMysterybox: &esv1.NebiusMysteryboxProvider{
  768. APIDomain: apiDomain,
  769. Auth: esv1.NebiusAuth{
  770. ServiceAccountCreds: esmeta.SecretKeySelector{
  771. Name: keySecretName,
  772. Key: keySecretKey,
  773. },
  774. },
  775. },
  776. },
  777. },
  778. }
  779. }
  780. func createK8sSecret(ctx context.Context, t *testing.T, k8sClient k8sclient.Client, namespace, secretName, secretKey string, secretValue []byte) {
  781. err := k8sClient.Create(ctx, &corev1.Secret{
  782. ObjectMeta: metav1.ObjectMeta{
  783. Namespace: namespace,
  784. Name: secretName,
  785. },
  786. Data: map[string][]byte{secretKey: secretValue},
  787. })
  788. tassert.NoError(t, err)
  789. }
  790. func unmarshalStringMap(t *testing.T, data []byte) map[string]string {
  791. stringMap := make(map[string]string)
  792. err := json.Unmarshal(data, &stringMap)
  793. tassert.NoError(t, err)
  794. return stringMap
  795. }
  796. func newProvider(t *testing.T, newMysteryboxClientFunc NewMysteryboxClient, tokenGetter TokenGetter) *Provider {
  797. t.Helper()
  798. cache, err := lru.New(10)
  799. tassert.NoError(t, err)
  800. return &Provider{
  801. Logger: logger,
  802. NewMysteryboxClient: newMysteryboxClientFunc,
  803. mysteryboxClientsCache: cache,
  804. TokenGetter: tokenGetter,
  805. }
  806. }
  807. type faketokenGetter struct {
  808. calls int32
  809. returnError bool
  810. gotDomain string
  811. gotCreds string
  812. gotCACert []byte
  813. tokenToIssue string
  814. mu sync.Mutex
  815. }
  816. func (f *faketokenGetter) GetToken(_ context.Context, apiDomain, subjectCreds string, caCert []byte) (string, error) {
  817. atomic.AddInt32(&f.calls, 1)
  818. f.mu.Lock()
  819. defer f.mu.Unlock()
  820. f.gotDomain = apiDomain
  821. f.gotCreds = subjectCreds
  822. f.gotCACert = caCert
  823. if f.returnError {
  824. return "", errors.New("internal error")
  825. }
  826. return f.tokenToIssue, nil
  827. }
  828. func setCacheSizeWorkaround(t *testing.T, size int, p *Provider) {
  829. t.Helper()
  830. err := p.initMysteryboxClientsCache()
  831. tassert.NoError(t, err)
  832. p.mysteryboxClientsCache.Resize(size)
  833. }
  834. func settokenGetterWorkaround(tokenGetter TokenGetter, p *Provider) {
  835. p.tokenInitMutex.Lock()
  836. defer p.tokenInitMutex.Unlock()
  837. p.TokenGetter = tokenGetter
  838. }
  839. type ClientData struct {
  840. domain string
  841. ca []byte
  842. }