feat: introduce priorityPolicy in merge rewrite (#5329)

* introduce priorityPolicy in merge rewrite

Signed-off-by: Riccardo M. Cefala <riccardo.c@miro.com>

* make reviewable

Signed-off-by: Riccardo M. Cefala <riccardo.c@miro.com>

* update documentation for priorityPolicy

Signed-off-by: Riccardo M. Cefala <riccardo.cefala@gmail.com>

* add some missing kubebuilder Enum validation directives

Signed-off-by: Riccardo M. Cefala <riccardo.c@miro.com>

---------

Signed-off-by: Riccardo M. Cefala <riccardo.c@miro.com>
Signed-off-by: Riccardo M. Cefala <riccardo.cefala@gmail.com>

apis/externalsecrets/v1/externalsecret_types.go

@@ -340,6 +340,11 @@ type ExternalSecretRewriteMerge struct {
 	// +optional
 	Priority []string `json:"priority,omitempty"`
 
+	// Used to define the policy when a key in the priority list does not exist in the input.
+	// +optional
+	// +kubebuilder:default="Strict"
+	PriorityPolicy ExternalSecretRewriteMergePriorityPolicy `json:"priorityPolicy,omitempty"`
+
 	// Used to define the policy to use in conflict resolution.
 	// +optional
 	// +kubebuilder:default="Error"
@@ -351,6 +356,7 @@ type ExternalSecretRewriteMerge struct {
 	Strategy ExternalSecretRewriteMergeStrategy `json:"strategy,omitempty"`
 }
 
+// +kubebuilder:validation:Enum=Ignore;Error
 type ExternalSecretRewriteMergeConflictPolicy string
 
 const (
@@ -358,6 +364,15 @@ const (
 	ExternalSecretRewriteMergeConflictPolicyError  ExternalSecretRewriteMergeConflictPolicy = "Error"
 )
 
+// +kubebuilder:validation:Enum=IgnoreNotFound;Strict
+type ExternalSecretRewriteMergePriorityPolicy string
+
+const (
+	ExternalSecretRewriteMergePriorityPolicyIgnoreNotFound ExternalSecretRewriteMergePriorityPolicy = "IgnoreNotFound"
+	ExternalSecretRewriteMergePriorityPolicyStrict         ExternalSecretRewriteMergePriorityPolicy = "Strict"
+)
+
+// +kubebuilder:validation:Enum=Extract;JSON
 type ExternalSecretRewriteMergeStrategy string
 
 const (
@@ -499,6 +514,7 @@ type GeneratorRef struct {
 	Name string `json:"name"`
 }
 
+// +kubebuilder:validation:Enum=Ready;Deleted
 type ExternalSecretConditionType string
 
 const (

config/crds/bases/external-secrets.io_clusterexternalsecrets.yaml

@@ -315,6 +315,9 @@ spec:
                                     default: Error
                                     description: Used to define the policy to use
                                       in conflict resolution.
+                                    enum:
+                                    - Ignore
+                                    - Error
                                     type: string
                                   into:
                                     default: ""
@@ -328,10 +331,22 @@ spec:
                                     items:
                                       type: string
                                     type: array
+                                  priorityPolicy:
+                                    default: Strict
+                                    description: Used to define the policy when a
+                                      key in the priority list does not exist in the
+                                      input.
+                                    enum:
+                                    - IgnoreNotFound
+                                    - Strict
+                                    type: string
                                   strategy:
                                     default: Extract
                                     description: Used to define the strategy to use
                                       in the merge operation.
+                                    enum:
+                                    - Extract
+                                    - JSON
                                     type: string
                                 type: object
                               regexp:

config/crds/bases/external-secrets.io_externalsecrets.yaml

@@ -295,6 +295,9 @@ spec:
                                 default: Error
                                 description: Used to define the policy to use in conflict
                                   resolution.
+                                enum:
+                                - Ignore
+                                - Error
                                 type: string
                               into:
                                 default: ""
@@ -308,10 +311,21 @@ spec:
                                 items:
                                   type: string
                                 type: array
+                              priorityPolicy:
+                                default: Strict
+                                description: Used to define the policy when a key
+                                  in the priority list does not exist in the input.
+                                enum:
+                                - IgnoreNotFound
+                                - Strict
+                                type: string
                               strategy:
                                 default: Extract
                                 description: Used to define the strategy to use in
                                   the merge operation.
+                                enum:
+                                - Extract
+                                - JSON
                                 type: string
                             type: object
                           regexp:
@@ -652,6 +666,9 @@ spec:
                     status:
                       type: string
                     type:
+                      enum:
+                      - Ready
+                      - Deleted
                       type: string
                   required:
                   - status

deploy/crds/bundle.yaml

@@ -299,6 +299,9 @@ spec:
                                     conflictPolicy:
                                       default: Error
                                       description: Used to define the policy to use in conflict resolution.
+                                      enum:
+                                        - Ignore
+                                        - Error
                                       type: string
                                     into:
                                       default: ""
@@ -311,9 +314,19 @@ spec:
                                       items:
                                         type: string
                                       type: array
+                                    priorityPolicy:
+                                      default: Strict
+                                      description: Used to define the policy when a key in the priority list does not exist in the input.
+                                      enum:
+                                        - IgnoreNotFound
+                                        - Strict
+                                      type: string
                                     strategy:
                                       default: Extract
                                       description: Used to define the strategy to use in the merge operation.
+                                      enum:
+                                        - Extract
+                                        - JSON
                                       type: string
                                   type: object
                                 regexp:
@@ -11682,6 +11695,9 @@ spec:
                                 conflictPolicy:
                                   default: Error
                                   description: Used to define the policy to use in conflict resolution.
+                                  enum:
+                                    - Ignore
+                                    - Error
                                   type: string
                                 into:
                                   default: ""
@@ -11694,9 +11710,19 @@ spec:
                                   items:
                                     type: string
                                   type: array
+                                priorityPolicy:
+                                  default: Strict
+                                  description: Used to define the policy when a key in the priority list does not exist in the input.
+                                  enum:
+                                    - IgnoreNotFound
+                                    - Strict
+                                  type: string
                                 strategy:
                                   default: Extract
                                   description: Used to define the strategy to use in the merge operation.
+                                  enum:
+                                    - Extract
+                                    - JSON
                                   type: string
                               type: object
                             regexp:
@@ -12027,6 +12053,9 @@ spec:
                       status:
                         type: string
                       type:
+                        enum:
+                          - Ready
+                          - Deleted
                         type: string
                     required:
                       - status

docs/api/spec.md

@@ -4215,6 +4215,20 @@ Required if strategy is JSON. Ignored otherwise.</p>
 </tr>
 <tr>
 <td>
+<code>priorityPolicy</code></br>
+<em>
+<a href="#external-secrets.io/v1.ExternalSecretRewriteMergePriorityPolicy">
+ExternalSecretRewriteMergePriorityPolicy
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Used to define the policy when a key in the priority list does not exist in the input.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>conflictPolicy</code></br>
 <em>
 <a href="#external-secrets.io/v1.ExternalSecretRewriteMergeConflictPolicy">
@@ -4264,6 +4278,27 @@ ExternalSecretRewriteMergeStrategy
 <td></td>
 </tr></tbody>
 </table>
+<h3 id="external-secrets.io/v1.ExternalSecretRewriteMergePriorityPolicy">ExternalSecretRewriteMergePriorityPolicy
+(<code>string</code> alias)</p></h3>
+<p>
+(<em>Appears on:</em>
+<a href="#external-secrets.io/v1.ExternalSecretRewriteMerge">ExternalSecretRewriteMerge</a>)
+</p>
+<p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Value</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody><tr><td><p>&#34;IgnoreNotFound&#34;</p></td>
+<td></td>
+</tr><tr><td><p>&#34;Strict&#34;</p></td>
+<td></td>
+</tr></tbody>
+</table>
 <h3 id="external-secrets.io/v1.ExternalSecretRewriteMergeStrategy">ExternalSecretRewriteMergeStrategy
 (<code>string</code> alias)</p></h3>
 <p>

docs/guides/datafrom-rewrite.md

@@ -18,10 +18,12 @@ The `Extract` strategy interprets all secret values in the secret map as JSON an
 
 The `JSON` strategy interprets all secret values in the secret map as JSON and merges all contained key/value pairs in the key specified by the _required_ parameter `into`. If the key specified by `into` already exists in the original secrets map it will be overwritten.
 
-Key collisions can be ignored or cause an error according to `conflictPolicy` which can be either `Ignore` or `Error`.  
+Key collisions can be ignored or cause an error according to `conflictPolicy` which can be either `Ignore` or `Error`.
 
 To guarantee deterministic results of the merge operation, secret keys are processed in alphabetical order. Key priority can also be made explicit by providing a list of secret keys in the `priority` parameter. These keys will be processed last in the order they appear while all other keys will still be processed in alphabetical order.
 
+Specifying a key in the `priority` list which is not found in the source secret will cause an error. You can override this behavior setting `priorityPolicy` to `IgnoreNotFound` instead of the default `Strict`.
+
 ## Considerations about Rewrite implementation
 
 1. The input of a subsequent rewrite operation are the outputs of the previous rewrite.
@@ -105,7 +107,7 @@ data:
     foo_baz: MjIyMg== #2222
 ```
 
-### Merging all secrets 
+### Merging all secrets
 
 The following ExternalSecret:
 ```yaml

pkg/utils/utils.go

@@ -164,6 +164,9 @@ func merge(operation esv1.ExternalSecretRewriteMerge, in map[string][]byte) (map
 	for _, key := range keys {
 		value, exists := in[key]
 		if !exists {
+			if operation.PriorityPolicy == esv1.ExternalSecretRewriteMergePriorityPolicyIgnoreNotFound {
+				continue
+			}
 			return nil, nil, fmt.Errorf("merge failed with key %q not found in input map", key)
 		}
 		var jsonMap map[string]any

pkg/utils/utils_test.go

@@ -714,7 +714,7 @@ func TestRewriteMerge(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "using priority with keys not in input",
+			name: "using priority with keys not in input (default strict)",
 			args: args{
 				operation: esv1.ExternalSecretRewriteMerge{
 					ConflictPolicy: esv1.ExternalSecretRewriteMergeConflictPolicyIgnore,
@@ -729,6 +729,27 @@ func TestRewriteMerge(t *testing.T) {
 			wantErr: true,
 		},
 		{
+			name: "using priority with keys not in input and ignore policy",
+			args: args{
+				operation: esv1.ExternalSecretRewriteMerge{
+					ConflictPolicy: esv1.ExternalSecretRewriteMergeConflictPolicyIgnore,
+					Priority:       []string{"non-existent-key", "mongo-credentials"},
+					PriorityPolicy: esv1.ExternalSecretRewriteMergePriorityPolicyIgnoreNotFound,
+				},
+				in: map[string][]byte{
+					"mongo-credentials": []byte(`{"username": "foz", "password": "baz"}`),
+					"redis-credentials": []byte(`{"host": "redis.example.com", "port": "6379"}`),
+				},
+			},
+			want: map[string][]byte{
+				"username": []byte("foz"),
+				"password": []byte("baz"),
+				"host":     []byte("redis.example.com"),
+				"port":     []byte("6379"),
+			},
+			wantErr: false,
+		},
+		{
 			name: "using conflict policy error",
 			args: args{
 				operation: esv1.ExternalSecretRewriteMerge{