Bug 1720752 - Fix stripe import for custom_field array (#2193)

* Add failing test for custom_fields in invoice object

* bug 1720752 - Fix failed stripe import due to custom_fields

bigquery_etl/stripe/allowed_fields.py

@@ -3,7 +3,7 @@
 import re
 from hashlib import sha256
 from pathlib import Path
-from typing import Any, Tuple, Type
+from typing import Any, Tuple, Type, List
 
 import click
 import stripe
@@ -89,6 +89,18 @@ class FilteredSchema:
         """Format stripe object for BigQuery, and validate against original schema."""
         return self._format_helper(row, self.allowed, self.root, (self.type,))
 
+    def _hash_kv_userid(self, arr: List[dict], key_name: str) -> List[dict]:
+        """Hash a userid key if it exists in a key-value list."""
+        res = []
+        for field in arr:
+            key, value = field[key_name], field["value"]
+            if key == "userid":
+                # hash fxa uid before it reaches BigQuery
+                key = "fxa_uid"
+                value = sha256(value.encode("UTF-8")).hexdigest()
+            res.append({key_name: key, "value": value})
+        return res
+
     def _format_helper(
         self,
         obj: Any,
@@ -97,12 +109,16 @@ class FilteredSchema:
         path: Tuple[str, ...],
         is_list_item: bool = False,
     ) -> Any:
-        if path[-1] in ("metadata", "custom_fields"):
+        if path[-1] == "metadata":
-            if "userid" in obj:
-                # hash fxa uid before it reaches BigQuery
-                obj["fxa_uid"] = sha256(obj.pop("userid").encode("UTF-8")).hexdigest()
             # format metadata as a key-value list
-            obj = [{"key": key, "value": value} for key, value in obj.items()]
+            obj = self._hash_kv_userid(
+                [{"key": key, "value": value} for key, value in obj.items()], "key"
+            )
+        elif path[-1] == "custom_fields":
+            # NOTE: We treat the custom field like metadata, but does the
+            # invoice custom field contain the userid?
+            # https://stripe.com/docs/api/invoices/create#create_invoice-custom_fields
+            obj = self._hash_kv_userid(obj, "name")
         if isinstance(obj, list):
             # enforce schema
             if field.mode != "REPEATED":

tests/stripe/test_allowed_fields.py

@@ -64,6 +64,21 @@ def test_format_row():
     }
     assert filtered_schema.format_row(dispute) == dispute
 
+    filtered_schema = FilteredSchema(stripe.Invoice)
+    invoice = {
+        "amount_due": 5,
+        "custom_fields": [
+            {"name": "userid", "value": "raw_user_id"},
+        ],
+    }
+    expect = {
+        "amount_due": 5,
+        "custom_fields": [
+            {"name": "fxa_uid", "value": sha256(b"raw_user_id").hexdigest()}
+        ],
+    }
+    assert filtered_schema.format_row(invoice) == expect
+
 
 @pytest.mark.parametrize(
     "row,message",