Merge branch 'keyko-io:main' into main

.github/workflows/tests.yml

@@ -20,7 +20,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pytest dict_digger requests
+        pip install pytest
+        pip install -e .
     - name: Tests
       run: |
-        python -m pytest tests/unit -v
+        python -m pytest -v

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.3.3](https://github.com/keyko-io/defi-crawler-py/compare/v0.3.2...v0.3.3)
+
+> 20 October 2021
+
+- add cream chains [`#61`](https://github.com/keyko-io/defi-crawler-py/pull/61)
+- dex protocols [`#60`](https://github.com/keyko-io/defi-crawler-py/pull/60)
+- add sushi celo [`#59`](https://github.com/keyko-io/defi-crawler-py/pull/59)
+- add aave avalanche [`#58`](https://github.com/keyko-io/defi-crawler-py/pull/58)
+- add dodoex  [`#57`](https://github.com/keyko-io/defi-crawler-py/pull/57)
+- add pangolin and traderjoe support [`#53`](https://github.com/keyko-io/defi-crawler-py/pull/53)
+- retry on calls [`#52`](https://github.com/keyko-io/defi-crawler-py/pull/52)
+- add dodoex swaps [`fbd5daa`](https://github.com/keyko-io/defi-crawler-py/commit/fbd5daa040ac3477dcd4d82ec5c8d1348377fbcf)
+- fix tests [`871264b`](https://github.com/keyko-io/defi-crawler-py/commit/871264bacf7c47646f57abdc0b33a664546e4890)
+- Adding v0.3.2 Changelog updates [`16e8b8a`](https://github.com/keyko-io/defi-crawler-py/commit/16e8b8aa07b5ecc6d24df3e994a8f6cb05f6459a)
+
 #### [v0.3.2](https://github.com/keyko-io/defi-crawler-py/compare/v0.3.1...v0.3.2)
 
 > 4 October 2021

README.md

@@ -109,17 +109,21 @@ Name | Type | Version | Chain
 --------|-------|---------|-------
 Aave | Lending | 2 | Ethereum
 Aave | Lending | 2 | Polygon
+Aave | Lending | 2 | Avalanche
 Compound | Lending | 2 | Ethereum
 Cream | Lending | 2 | Ethereum
 Cream | Lending | 2 | BSC
 Cream | Lending | 2 | Polygon
 Cream | Lending | 2 | Arbitrum
+Cream | Lending | 2 | Avalance
+Cream | Lending | 2 | Fantom
 Kashi | Lending | 1 | Ethereum
 Kashi | Lending | 1 | Polygon
 Kashi | Lending | 1 | xDai
 Kashi | Lending | 1 | fantom
 Uniswap | Dexes | 2 | Ethereum
 Uniswap | Dexes | 3 | Ethereum
+Uniswap | Dexes | 3 | Optimism
 Balancer | Dexes | 2 | Ethereum
 Balancer | Dexes | 2 | Arbitrum
 Balancer | Dexes | 2 | Polygon
@@ -128,8 +132,15 @@ SushiSwap | Dexes | 1 | Ethereum
 SushiSwap | Dexes | 1 | BSC
 SushiSwap | Dexes | 1 | Polygon
 SushiSwap | Dexes | 1 | Fantom
+SushiSwap | Dexes | 1 | Celo
+Dodoex | Dexes | 2 | Ethereum
+Dodoex | Dexes | 2 | Polygon
+Dodoex | Dexes | 2 | BSC
+Dodoex | Dexes | 2 | Arbitrum
 Ubeswap | Dexes | 1 | Celo
 Pancakeswap | Dexes | 2 | BSC
+Pangolin | Dexes | 1 | Avalanche
+Traderjoe | Dexes | 1 | Avalanche
 
 
 

deficrawler/api_calls.py

@@ -2,6 +2,7 @@
 
 import requests
 import json
+from retry import retry
 
 
 def get_data_from(query_input, entity, from_timestamp, to_timestamp, mappings_file, endpoint, aditional_filters=""):
@@ -28,29 +29,24 @@ def get_data_from(query_input, entity, from_timestamp, to_timestamp, mappings_fi
             aditional_filters=filters_str
         )
 
-        response = requests.post(endpoint, json={'query': query})
-        json_data = json.loads(response.text)
-        if 'errors' in json_data:
-            raise Exception(
-                'There was an error getting the data from TheGraph' + json.dumps(json_data))
-        else:
-            response_lenght = len(json_data['data'][entity_name])
-            if (response_lenght > 0):
-                json_data = json.loads(response.text)
-                list_data = json_data['data'][entity_name]
-                iteration_timestamp = json_data['data'][entity_name][response_lenght - 1][order_by]
+        json_data = call_api(endpoint=endpoint, query=query)
 
-                json_records = [*json_records, *list_data]
-            else:
-                are_data = False
+        response_lenght = len(json_data['data'][entity_name])
+        if (response_lenght > 0):
+            list_data = json_data['data'][entity_name]
+            iteration_timestamp = json_data['data'][entity_name][response_lenght - 1][order_by]
+
+            json_records = [*json_records, *list_data]
+        else:
+            are_data = False
 
     return json_records
 
 
 def get_data_parameter(query_input, entity, mappings_file, endpoint):
     """
     Gets all the existing data for the given entity.
-    If this entity has some filter, in the config file, the query will apply 
+    If this entity has some filter, in the config file, the query will apply
     this filter
     """
     are_data = True
@@ -69,21 +65,16 @@ def get_data_parameter(query_input, entity, mappings_file, endpoint):
             attributes=attributes
         )
 
-        response = requests.post(endpoint, json={'query': query})
-        json_data = json.loads(response.text)
-        if 'errors' in json_data:
-            raise Exception(
-                'There was an error getting the data from TheGraph' + json.dumps(json_data))
-        else:
-            response_lenght = len(json_data['data'][entity_name])
-            if (response_lenght > 0):
-                json_data = json.loads(response.text)
-                list_data = json_data['data'][entity_name]
-                filter_value = json_data['data'][entity_name][response_lenght - 1][order_by]
+        json_data = call_api(endpoint=endpoint, query=query)
+
+        response_lenght = len(json_data['data'][entity_name])
+        if (response_lenght > 0):
+            list_data = json_data['data'][entity_name]
+            filter_value = json_data['data'][entity_name][response_lenght - 1][order_by]
 
-                json_records = [*json_records, *list_data]
-            else:
-                are_data = False
+            json_records = [*json_records, *list_data]
+        else:
+            are_data = False
 
     return json_records
 
@@ -103,19 +94,14 @@ def get_data_filtered(query_input, entity, mappings_file, endpoint, filters):
         attributes=attributes
     )
 
-    response = requests.post(endpoint, json={'query': query})
-    json_data = json.loads(response.text)
+    json_data = call_api(endpoint=endpoint, query=query)
     json_records = []
-    if 'errors' in json_data:
-        raise Exception(
-            'There was an error getting the data from TheGraph' + json.dumps(json_data))
-    else:
-        response_lenght = len(json_data['data'][entity_name])
-        if (response_lenght > 0):
-            json_data = json.loads(response.text)
-            list_data = json_data['data'][entity_name]
 
-            json_records = [*list_data]
+    response_lenght = len(json_data['data'][entity_name])
+    if (response_lenght > 0):
+        list_data = json_data['data'][entity_name]
+
+        json_records = [*list_data]
 
     return json_records
 
@@ -144,18 +130,27 @@ def get_first_element(query_input, entity, mappings_file, endpoint, timestamp, a
         block=params['block']
     )
 
+    json_data = call_api(endpoint=endpoint, query=query)
+    json_records = []
+
+    response_lenght = len(json_data['data'][entity_name])
+    if (response_lenght > 0):
+        list_data = json_data['data'][entity_name]
+
+        json_records = [*list_data]
+
+    return json_records
+
+@retry(Exception, tries=3, delay=2)
+def call_api(endpoint, query):
+    """
+    Returns the data from the api, if there is any error throws
+    """
+
     response = requests.post(endpoint, json={'query': query})
     json_data = json.loads(response.text)
-    json_records = []
     if 'errors' in json_data:
         raise Exception(
             'There was an error getting the data from TheGraph' + json.dumps(json_data))
-    else:
-        response_lenght = len(json_data['data'][entity_name])
-        if (response_lenght > 0):
-            json_data = json.loads(response.text)
-            list_data = json_data['data'][entity_name]
 
-            json_records = [*list_data]
-
-    return json_records
+    return json_data

deficrawler/config/aave-2.json

@@ -6,7 +6,8 @@
     "chain": "Ethereum",
     "endpoint": {
       "ethereum": "https://api.thegraph.com/subgraphs/name/aave/protocol-v2",
-      "polygon": "https://api.thegraph.com/subgraphs/name/aave/aave-v2-matic"
+      "polygon": "https://api.thegraph.com/subgraphs/name/aave/aave-v2-matic",
+      "avalanche": "https://api.thegraph.com/subgraphs/name/aave/protocol-v2-avalanche"
     }
   },
   "entities": {

deficrawler/config/compound-2.json

@@ -5,7 +5,7 @@
     "version": 2,
     "chain": "Ethereum",
     "endpoint": {
-      "ethereum": "https://api.thegraph.com/subgraphs/name/graphprotocol/compound-v2"
+      "ethereum": "https://api.thegraph.com/subgraphs/name/cavanmflynn/compound-v2"
     }
   },
   "entities": {