Azure
/
azure-functions-python-library
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

json -> ujson move to test speeding up! Trying this out.

This commit is contained in:
Varad Meru [gmail] 2020-12-19 01:53:09 -08:00
Родитель 483b395a44
Коммит e45fe64233
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: D65E2959EB74910D
18 изменённых файлов: 87 добавлений и 83 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

6
azure/functions/_cosmosdb.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
import collections
import json
import ujson
from . import _abc
@ -20,7 +20,7 @@ class Document(_abc.Document, collections.UserDict):
@classmethod
def from_json(cls, json_data: str) -> 'Document':
"""Create a Document from a JSON string."""
return cls.from_dict(json.loads(json_data))
return cls.from_dict(ujson.loads(json_data))
@classmethod
def from_dict(cls, dct: dict) -> 'Document':
@ -30,7 +30,7 @@ class Document(_abc.Document, collections.UserDict):
def to_json(self) -> str:
"""Return the JSON representation of the document."""
return json.dumps(dict(self))
return ujson.dumps(dict(self))
def __getitem__(self, key):
return collections.UserDict.__getitem__(self, key)

4
azure/functions/_http.py
Просмотреть файл

@ -3,7 +3,7 @@
import collections.abc
import io
import json
import ujson
import typing
import types
@ -203,7 +203,7 @@ class HttpRequest(_abc.HttpRequest):
return self.__body_bytes
def get_json(self) -> typing.Any:
return json.loads(self.__body_bytes.decode('utf-8'))
return ujson.loads(self.__body_bytes.decode('utf-8'))
def _parse_form_data(self):
if self.__form_parsed:

4
azure/functions/_queue.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
import datetime
import json
import ujson
import typing
from . import _abc
@ -86,7 +86,7 @@ class QueueMessage(_abc.QueueMessage):
:raises ValueError:
when the body of the message does not contain valid JSON data.
"""
return json.loads(self.__body)
return ujson.loads(self.__body)
def __repr__(self) -> str:
return (

6
azure/functions/cosmosdb.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
import collections.abc
import json
import ujson
import typing
from azure.functions import _cosmosdb as cdb
@ -41,7 +41,7 @@ class CosmosDBConverter(meta.InConverter, meta.OutConverter,
raise NotImplementedError(
f'unsupported queue payload type: {data_type}')
documents = json.loads(body)
documents = ujson.loads(body)
if not isinstance(documents, list):
documents = [documents]
@ -72,7 +72,7 @@ class CosmosDBConverter(meta.InConverter, meta.OutConverter,
return meta.Datum(
type='json',
value=json.dumps([dict(d) for d in data])
value=ujson.dumps([dict(d) for d in data])
)

8
azure/functions/eventgrid.py
Просмотреть файл

@ -3,7 +3,7 @@
import collections
import datetime
import json
import ujson
from typing import Optional, List, Any, Dict, Union
from azure.functions import _eventgrid as azf_eventgrid
@ -31,7 +31,7 @@ class EventGridEventInConverter(meta.InConverter, binding='eventGridTrigger',
data_type = data.type
if data_type == 'json':
body = json.loads(data.value)
body = ujson.loads(data.value)
else:
raise NotImplementedError(
f'unsupported event grid payload type: {data_type}')
@ -70,7 +70,7 @@ class EventGridEventOutConverter(meta.OutConverter, binding="eventGrid"):
elif isinstance(obj, azf_eventgrid.EventGridOutputEvent):
return meta.Datum(
type='json',
value=json.dumps({
value=ujson.dumps({
'id': obj.id,
'subject': obj.subject,
'dataVersion': obj.data_version,
@ -101,7 +101,7 @@ class EventGridEventOutConverter(meta.OutConverter, binding="eventGrid"):
return meta.Datum(
type='json',
value=json.dumps(msgs)
value=ujson.dumps(msgs)
)
raise NotImplementedError

12
azure/functions/eventhub.py
Просмотреть файл

@ -1,7 +1,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import ujson
from typing import Dict, Any, List, Union, Optional, Mapping
from azure.functions import _eventhub
@ -88,7 +88,7 @@ class EventHubConverter(meta.InConverter, meta.OutConverter,
data = meta.Datum(type='int', value=obj)
elif isinstance(obj, list):
data = meta.Datum(type='json', value=json.dumps(obj))
data = meta.Datum(type='json', value=ujson.dumps(obj))
return data
@ -147,13 +147,13 @@ class EventHubTriggerConverter(EventHubConverter,
# Input Trigger IotHub Event
elif data.type == 'json':
parsed_data = json.loads(data.value)
parsed_data = ujson.loads(data.value)
sys_props = trigger_metadata.get('SystemPropertiesArray')
parsed_sys_props: List[Any] = []
if sys_props is not None:
parsed_sys_props = json.loads(sys_props.value)
parsed_sys_props = ujson.loads(sys_props.value)
if len(parsed_data) != len(parsed_sys_props):
raise AssertionError('Number of bodies and metadata mismatched')
@ -198,7 +198,7 @@ class EventHubTriggerConverter(EventHubConverter,
# it is handled as single_event by mistake and our users handle the
# data parsing. And we want to keep the same behavior here.
if data_type == 'json':
return json.dumps(parsed_data).encode('utf-8')
return ujson.dumps(parsed_data).encode('utf-8')
elif data_type == 'bytes':
return parsed_data
elif data_type == 'string':
@ -238,7 +238,7 @@ class EventHubTriggerConverter(EventHubConverter,
@classmethod
def _extract_iothub_from_system_properties(
cls, system_properties_string: str) -> Dict[str, str]:
system_properties = json.loads(system_properties_string)
system_properties = ujson.loads(system_properties_string)
return cls._extract_iothub_from_dict(system_properties)
@classmethod

6
azure/functions/http.py
Просмотреть файл

@ -1,7 +1,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import ujson
import typing
from azure.functions import _abc as azf_abc
@ -53,10 +53,10 @@ class HttpRequest(azf_http.HttpRequest):
def get_json(self) -> typing.Any:
if self.__body_type in ('json', 'string'):
assert self.__body_str is not None
return json.loads(self.__body_str)
return ujson.loads(self.__body_str)
elif self.__body_bytes is not None:
try:
return json.loads(self.__body_bytes.decode('utf-8'))
return ujson.loads(self.__body_bytes.decode('utf-8'))
except ValueError as e:
raise ValueError(
'HTTP request does not contain valid JSON data') from e

4
azure/functions/kafka.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
import typing
import json
import ujson
from typing import Any, List
@ -269,7 +269,7 @@ class KafkaTriggerConverter(KafkaConverter,
cls, props: meta.Datum, parsed_data) -> List[Any]:
parsed_props: List[Any] = []
if props is not None:
parsed_props = json.loads(props.value)
parsed_props = ujson.loads(props.value)
if len(parsed_data) != len(parsed_props):
raise AssertionError('Number of bodies and metadata mismatched')
return parsed_props

6
azure/functions/meta.py
Просмотреть файл

@ -4,7 +4,7 @@
import abc
import collections.abc
import datetime
import json
import ujson
import re
from typing import Dict, Optional, Union, Tuple, Mapping, Any
@ -49,7 +49,7 @@ class Datum:
elif self.type in ('bytes', 'string', 'int', 'double'):
return self.value
elif self.type == 'json':
return json.loads(self.value)
return ujson.loads(self.value)
elif self.type == 'collection_string':
return [v for v in self.value.string]
elif self.type == 'collection_bytes':
@ -125,7 +125,7 @@ class _BaseConverter(metaclass=_ConverterMeta, binding=None):
data_type = data.type
if data_type == 'json':
result = json.loads(data.value)
result = ujson.loads(data.value)
elif data_type == 'string':
result = data.value

6
azure/functions/queue.py
Просмотреть файл

@ -3,7 +3,7 @@
import collections.abc
import datetime
import json
import ujson
from typing import List, Dict, Any, Union, Optional
from azure.functions import _abc as azf_abc
@ -116,7 +116,7 @@ class QueueMessageOutConverter(meta.OutConverter, binding='queue'):
elif isinstance(obj, azf_queue.QueueMessage):
return meta.Datum(
type='json',
value=json.dumps({
value=ujson.dumps({
'id': obj.id,
'body': obj.get_body().decode('utf-8'),
})
@ -139,7 +139,7 @@ class QueueMessageOutConverter(meta.OutConverter, binding='queue'):
return meta.Datum(
type='json',
value=json.dumps(msgs)
value=ujson.dumps(msgs)
)
raise NotImplementedError

12
azure/functions/servicebus.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
import datetime
import json
import ujson
from typing import Dict, Any, List, Union, Optional, Mapping, cast
from azure.functions import _servicebus as azf_sbus
@ -298,7 +298,7 @@ class ServiceBusMessageInConverter(meta.InConverter,
# Input Trigger IotHub Event
elif data.type == 'json':
parsed_data = json.loads(data.value)
parsed_data = ujson.loads(data.value)
else:
raise NotImplementedError('unable to decode multiple messages '
@ -326,7 +326,7 @@ class ServiceBusMessageInConverter(meta.InConverter,
trigger_metadata: Mapping[str, meta.Datum]
) -> int:
datum = trigger_metadata['UserPropertiesArray']
user_props = json.loads(datum.value)
user_props = ujson.loads(datum.value)
return len(user_props)
@classmethod
@ -340,7 +340,7 @@ class ServiceBusMessageInConverter(meta.InConverter,
elif data_type == 'str' and isinstance(body, str):
return body.encode('utf-8')
elif data_type == 'json' and isinstance(body, dict):
return json.dumps(body).encode('utf-8')
return ujson.dumps(body).encode('utf-8')
else:
raise NotImplementedError('unable to marshall message body with '
f'data_type {data_type}')
@ -361,7 +361,7 @@ class ServiceBusMessageInConverter(meta.InConverter,
return cast(List[bytes], [b.encode('utf-8') for b in strings])
elif data_type == 'json':
return cast(List[bytes],
[json.dumps(b).encode('utf-8') for b in bodies])
[ujson.dumps(b).encode('utf-8') for b in bodies])
else:
raise NotImplementedError('unable to marshall message bodies with '
f'data_type {data_type}')
@ -468,7 +468,7 @@ class ServiceBusMessageInConverter(meta.InConverter,
elif datum.type == 'collection_sint64':
data_array = datum.value.sint64
elif datum.type == 'json':
data_array = json.loads(datum.value)
data_array = ujson.loads(datum.value)
# Check if the index is inbound
if data_array is None or index >= len(data_array):

4
azure/functions/timer.py
Просмотреть файл

@ -1,7 +1,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import ujson
import typing
from azure.functions import _abc as azf_abc
@ -31,6 +31,6 @@ class TimerRequestConverter(meta.InConverter,
if data.type != 'json':
raise NotImplementedError
info = json.loads(data.value)
info = ujson.loads(data.value)
return TimerRequest(
past_due=info.get('IsPastDue', False))

3
setup.py
Просмотреть файл

@ -34,6 +34,9 @@ setup(
package_data={
'azure.functions': ['py.typed']
},
install_requires=[
'ujson~=4.0.1'
],
extras_require={
'dev': [
'flake8~=3.7.9',

10
tests/test_blob.py
Просмотреть файл

@ -1,6 +1,6 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import ujson
import unittest
from typing import Any, Dict
@ -100,7 +100,7 @@ class TestBlob(unittest.TestCase):
self.assertEqual(result.length, len(b'blob_content'))
self.assertEqual(result.uri, 'https://test.io/blob_trigger')
self.assertEqual(result.blob_properties,
json.loads(sample_blob_properties))
ujson.loads(sample_blob_properties))
self.assertEqual(result.metadata, None)
def test_blob_input_with_metadata_with_trigger_metadata(self):
@ -133,9 +133,9 @@ class TestBlob(unittest.TestCase):
self.assertEqual(result.length, len(b'blob_content'))
self.assertEqual(result.uri, 'https://test.io/blob_trigger')
self.assertEqual(result.blob_properties,
json.loads(sample_blob_properties))
ujson.loads(sample_blob_properties))
self.assertEqual(result.metadata,
json.loads(sample_metadata))
ujson.loads(sample_metadata))
def test_blob_input_with_metadata_with_incorrect_trigger_metadata(self):
sample_metadata = 'Hello World'
@ -156,7 +156,7 @@ class TestBlob(unittest.TestCase):
self.assertEqual(result.length, len(b'blob_content'))
self.assertEqual(result.uri, 'https://test.io/blob_trigger')
self.assertEqual(result.blob_properties,
json.loads(sample_blob_properties))
ujson.loads(sample_blob_properties))
self.assertEqual(result.metadata, None)
def test_blob_incomplete_read(self):

21
tests/test_eventhub.py
Просмотреть файл

@ -3,7 +3,7 @@
from typing import List, Mapping
import unittest
import json
import ujson
from unittest.mock import patch
from datetime import datetime
@ -113,12 +113,12 @@ class TestEventHub(unittest.TestCase):
self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[0].get_body().decode('utf-8'), '{"device-status": "good1"}'
result[0].get_body().decode('utf-8'), '{"device-status":"good1"}'
)
self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[1].get_body().decode('utf-8'), '{"device-status": "good2"}'
result[1].get_body().decode('utf-8'), '{"device-status":"good2"}'
)
def test_eventhub_trigger_multiple_events_collection_string(self):
@ -131,12 +131,12 @@ class TestEventHub(unittest.TestCase):
self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[0].get_body().decode('utf-8'), '{"device-status": "good1"}'
result[0].get_body().decode('utf-8'), '{"device-status":"good1"}'
)
self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[1].get_body().decode('utf-8'), '{"device-status": "good2"}'
result[1].get_body().decode('utf-8'), '{"device-status":"good2"}'
)
def test_eventhub_trigger_multiple_events_collection_bytes(self):
@ -149,12 +149,12 @@ class TestEventHub(unittest.TestCase):
self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[0].get_body().decode('utf-8'), '{"device-status": "good1"}'
result[0].get_body().decode('utf-8'), '{"device-status":"good1"}'
)
self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME)
self.assertEqual(
result[1].get_body().decode('utf-8'), '{"device-status": "good2"}'
result[1].get_body().decode('utf-8'), '{"device-status":"good2"}'
)
def test_iothub_metadata_events(self):
@ -299,12 +299,13 @@ class TestEventHub(unittest.TestCase):
data = '[{"device-status": "good1"}, {"device-status": "good2"}]'
if data_type == 'collection_bytes':
data = list(
map(lambda x: json.dumps(x).encode('utf-8'), json.loads(data))
map(lambda x: ujson.dumps(x).encode('utf-8'),
ujson.loads(data))
)
data = CollectionBytes(data)
elif data_type == 'collection_string':
data = list(
map(lambda x: json.dumps(x), json.loads(data))
map(lambda x: ujson.dumps(x), ujson.loads(data))
)
data = CollectionString(data)
@ -334,6 +335,6 @@ class TestEventHub(unittest.TestCase):
return {
'SystemPropertiesArray': meta.Datum(
json.dumps(system_props_array), 'json'
ujson.dumps(system_props_array), 'json'
)
}

36
tests/test_kafka.py
Просмотреть файл

@ -3,7 +3,7 @@
from typing import List
import unittest
import json
import ujson
from unittest.mock import patch
import azure.functions as func
@ -112,16 +112,16 @@ class Kafka(unittest.TestCase):
result[0].timestamp,
self.MULTIPLE_KAFKA_TIMESTAMP_0)
self.assertEqual(
json.loads(result[0].get_body().decode('utf-8')),
json.loads(self.MULTIPLE_KAFKA_DATA_0)
ujson.loads(result[0].get_body().decode('utf-8')),
ujson.loads(self.MULTIPLE_KAFKA_DATA_0)
)
self.assertEqual(
result[1].timestamp,
self.MULTIPLE_KAFKA_TIMESTAMP_1)
self.assertEqual(
json.loads(result[1].get_body().decode('utf-8')),
json.loads(self.MULTIPLE_KAFKA_DATA_1)
ujson.loads(result[1].get_body().decode('utf-8')),
ujson.loads(self.MULTIPLE_KAFKA_DATA_1)
)
def test_kafka_trigger_multiple_events_collection_bytes(self):
@ -136,16 +136,16 @@ class Kafka(unittest.TestCase):
result[0].timestamp,
self.MULTIPLE_KAFKA_TIMESTAMP_0)
self.assertEqual(
json.loads(result[0].get_body().decode('utf-8')),
json.loads(self.MULTIPLE_KAFKA_DATA_0)
ujson.loads(result[0].get_body().decode('utf-8')),
ujson.loads(self.MULTIPLE_KAFKA_DATA_0)
)
self.assertEqual(
result[1].timestamp,
self.MULTIPLE_KAFKA_TIMESTAMP_1)
self.assertEqual(
json.loads(result[1].get_body().decode('utf-8')),
json.loads(self.MULTIPLE_KAFKA_DATA_1)
ujson.loads(result[1].get_body().decode('utf-8')),
ujson.loads(self.MULTIPLE_KAFKA_DATA_1)
)
def test_single_kafka_trigger_metadata_field(self):
@ -164,11 +164,11 @@ class Kafka(unittest.TestCase):
self.assertEqual(result.partition, 0)
# Value
self.assertEqual(
json.loads(result.get_body().decode('utf-8'))['Value'], "hello")
ujson.loads(result.get_body().decode('utf-8'))['Value'], "hello")
# Metadata
metadata_dict = result.metadata
sys = metadata_dict['sys']
sys_dict = json.loads(sys)
sys_dict = ujson.loads(sys)
self.assertEqual(sys_dict['MethodName'], 'KafkaTrigger')
def test_multiple_kafka_triggers_metadata_field(self):
@ -191,7 +191,7 @@ class Kafka(unittest.TestCase):
"2020-06-20T05:06:25.945Z")
metadata_dict = result[0].metadata
sys = metadata_dict['sys']
sys_dict = json.loads(sys)
sys_dict = ujson.loads(sys)
self.assertEqual(sys_dict['MethodName'], 'KafkaTriggerMany')
def _generate_single_kafka_datum(self, datum_type='string'):
@ -207,13 +207,13 @@ class Kafka(unittest.TestCase):
'"Timestamp":"2020-06-20T05:06:25.945Z","Value":"a"}]'
if data_type == 'collection_bytes':
data = list(
map(lambda x: json.dumps(x).encode('utf-8'),
json.loads(data))
map(lambda x: ujson.dumps(x).encode('utf-8'),
ujson.loads(data))
)
data = CollectionBytes(data)
elif data_type == 'collection_string':
data = list(
map(lambda x: json.dumps(x), json.loads(data))
map(lambda x: ujson.dumps(x), ujson.loads(data))
)
data = CollectionString(data)
@ -252,16 +252,16 @@ class Kafka(unittest.TestCase):
return {
'KeyArray': meta.Datum(
json.dumps(key_array), 'json'
ujson.dumps(key_array), 'json'
),
'OffsetArray': meta.Datum(
CollectionSint64([62, 63]), 'collection_sint64'
),
'PartitionArray': meta.Datum(
json.dumps(partition_array), 'json'
ujson.dumps(partition_array), 'json'
),
'TimestampArray': meta.Datum(
json.dumps(timestamp_array), 'json'
ujson.dumps(timestamp_array), 'json'
),
'TopicArray': meta.Datum(
CollectionString(['message', 'message']), "collection_string"

4
tests/test_queue.py
Просмотреть файл

@ -1,7 +1,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import json
import ujson
import unittest
import azure.functions as func
@ -44,7 +44,7 @@ class TestQueue(unittest.TestCase):
def test_QueueMessage_get_json(self):
# given
test_body: str = '{"isJSON": "True"}'
expected_body = json.loads(test_body)
expected_body = ujson.loads(test_body)
# when
test_queue_message = func.QueueMessage(

18
tests/test_servicebus.py
Просмотреть файл

@ -2,7 +2,7 @@
# Licensed under the MIT License.
from typing import Dict, List
import json
import ujson
import unittest
from datetime import datetime, timedelta
@ -80,7 +80,7 @@ class TestServiceBus(unittest.TestCase):
trigger_metadata=self._generate_single_trigger_metadata())
servicebus_data = servicebus_msg.get_body().decode('utf-8')
self.assertEqual(servicebus_data, json.dumps({"lucky_number": 23}))
self.assertEqual(servicebus_data, ujson.dumps({"lucky_number": 23}))
def test_servicebus_non_existing_property(self):
# The function should not fail even when property does not work
@ -243,9 +243,9 @@ class TestServiceBus(unittest.TestCase):
)
expceted_bodies: List[str] = [
json.dumps({"lucky_number": 23}),
json.dumps({"lucky_number": 34}),
json.dumps({"lucky_number": 45}),
ujson.dumps({"lucky_number": 23}),
ujson.dumps({"lucky_number": 34}),
ujson.dumps({"lucky_number": 45}),
]
expected_message_ids: List[int] = [
@ -297,12 +297,12 @@ class TestServiceBus(unittest.TestCase):
})
def _generate_single_servicebus_data(self) -> meta.Datum:
return meta.Datum(value=json.dumps({
return meta.Datum(value=ujson.dumps({
'lucky_number': 23
}), type='json')
def _generate_multiple_service_bus_data(self) -> meta.Datum:
return meta.Datum(value=json.dumps([
return meta.Datum(value=ujson.dumps([
{'lucky_number': 23},
{'lucky_number': 34},
{'lucky_number': 45}
@ -508,9 +508,9 @@ class TestServiceBus(unittest.TestCase):
)
elif expected_type == 'json':
if datum_type == 'json':
value = json.dumps([json.loads(d[key].value) for d in args])
value = ujson.dumps([ujson.loads(d[key].value) for d in args])
else:
value = json.dumps([d[key].value for d in args])
value = ujson.dumps([d[key].value for d in args])
return meta.Datum(
value=value,
type='json'