Merge pull request #1126 from JakubOnderka/orjson

Add support for orjson

.github/workflows/pytest.yml

@@ -36,5 +36,11 @@ jobs:
         poetry run pytest --cov=pymisp tests/test_*.py
         poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp
 
+    - name: Test with nosetests with orjson
+      run: |
+        pip3 install orjson
+        poetry run pytest --cov=pymisp tests/test_*.py
+        poetry run mypy tests/testlive_comprehensive.py tests/test_mispevent.py tests/testlive_sync.py pymisp
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

README.md

@@ -125,7 +125,6 @@ logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', for
 # From poetry
 
 pytest --cov=pymisp tests/test_*.py tests/testlive_comprehensive.py:TestComprehensive.[test_name]
-
 ```
 
 ## Documentation
@@ -180,9 +179,9 @@ poetry build
 mv dist/*.whl offline/packages/
 ```
 
-2. Copy the content of `offline/packages/` to the machine with no internet access.
+3. Copy the content of `offline/packages/` to the machine with no internet access.
 
-3. Install the packages:
+4. Install the packages:
 
 ```bash
 python -m pip install --no-index --no-deps packages/*.whl

pymisp/abstract.py

@@ -1,53 +1,45 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
+import logging
 from datetime import date, datetime
-
 from deprecated import deprecated  # type: ignore
 from json import JSONEncoder
 from uuid import UUID
 from abc import ABCMeta
-
-try:
-    from rapidjson import load  # type: ignore
-    from rapidjson import loads  # type: ignore
-    from rapidjson import dumps  # type: ignore
-    HAS_RAPIDJSON = True
-except ImportError:
-    from json import load
-    from json import loads
-    from json import dumps
-    HAS_RAPIDJSON = False
-
-import logging
 from enum import Enum
 from typing import Union, Optional, Any, Dict, List, Set, Mapping
-
-from .exceptions import PyMISPInvalidFormat, PyMISPError
-
-
 from collections.abc import MutableMapping
 from functools import lru_cache
 from pathlib import Path
 
+try:
+    import orjson  # type: ignore
+    from orjson import loads, dumps  # type: ignore
+    HAS_ORJSON = True
+except ImportError:
+    from json import loads, dumps
+    HAS_ORJSON = False
+
+from .exceptions import PyMISPInvalidFormat, PyMISPError
+
 logger = logging.getLogger('pymisp')
 
+
 resources_path = Path(__file__).parent / 'data'
 misp_objects_path = resources_path / 'misp-objects' / 'objects'
-with (resources_path / 'describeTypes.json').open('r') as f:
-    describe_types = load(f)['result']
+with (resources_path / 'describeTypes.json').open('rb') as f:
+    describe_types = loads(f.read())['result']
 
 
 class MISPFileCache(object):
     # cache up to 150 JSON structures in class attribute
 
     @staticmethod
     @lru_cache(maxsize=150)
-    def _load_json(path: Path) -> Union[dict, None]:
+    def _load_json(path: Path) -> Optional[dict]:
         if not path.exists():
             return None
-        with path.open('r', encoding='utf-8') as f:
-            data = load(f)
+        with path.open('rb') as f:
+            data = loads(f.read())
         return data
 
 
@@ -249,6 +241,15 @@ def _to_feed(self) -> Dict:
 
     def to_json(self, sort_keys: bool = False, indent: Optional[int] = None) -> str:
         """Dump recursively any class of type MISPAbstract to a json string"""
+        if HAS_ORJSON:
+            option = 0
+            if sort_keys:
+                option |= orjson.OPT_SORT_KEYS
+            if indent:
+                option |= orjson.OPT_INDENT_2
+
+            return dumps(self, default=pymisp_json_default, option=option).decode("utf-8")
+
         return dumps(self, default=pymisp_json_default, sort_keys=sort_keys, indent=indent)
 
     def __getitem__(self, key):
@@ -406,23 +407,13 @@ def __repr__(self) -> str:
         return '<{self.__class__.__name__}(NotInitialized)>'.format(self=self)
 
 
-if HAS_RAPIDJSON:
-    def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]:
-        if isinstance(obj, AbstractMISP):
-            return obj.jsonable()
-        elif isinstance(obj, (datetime, date)):
-            return obj.isoformat()
-        elif isinstance(obj, Enum):
-            return obj.value
-        elif isinstance(obj, UUID):
-            return str(obj)
-else:
-    def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]:
-        if isinstance(obj, AbstractMISP):
-            return obj.jsonable()
-        elif isinstance(obj, (datetime, date)):
-            return obj.isoformat()
-        elif isinstance(obj, Enum):
-            return obj.value
-        elif isinstance(obj, UUID):
-            return str(obj)
+# UUID, datetime, date and Enum is serialized by ORJSON by default
+def pymisp_json_default(obj: Union[AbstractMISP, datetime, date, Enum, UUID]) -> Union[Dict, str]:
+    if isinstance(obj, AbstractMISP):
+        return obj.jsonable()
+    elif isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    elif isinstance(obj, Enum):
+        return obj.value
+    elif isinstance(obj, UUID):
+        return str(obj)