From e0c0e3bf2b5e2eceae68343e4ea631f318587cc1 Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Thu, 11 Feb 2021 16:50:41 +0530 Subject: [PATCH] feat: implement custom redis endpoint Signed-off-by: Ameya Shenoy --- backend/app/redis.py | 32 ++++++++++++++++++++++++++++++++ backend/app/urls.py | 14 ++++++++++++-- backend/app/utils.py | 32 +++++++++++++++++++++++++++++--- backend/app/views.py | 17 ++++++++++++++++- backend/backend/settings.py | 6 ++++++ frontend/src/axios-api.js | 2 +- frontend/src/views/BhavCopy.vue | 5 +++-- 7 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 backend/app/redis.py diff --git a/backend/app/redis.py b/backend/app/redis.py new file mode 100644 index 0000000..0905cc4 --- /dev/null +++ b/backend/app/redis.py @@ -0,0 +1,32 @@ + +import json + +from django.core.serializers.json import DjangoJSONEncoder +from django_redis.pool import ConnectionFactory as Factory +from django_redis.serializers.base import BaseSerializer + + +class JSONSerializer(BaseSerializer): + """ + The default JSON serializer of django-redis assumes `decode_responses` + is disabled, and calls `decode()` and `encode()` on the value. + This serializer does not. + """ + + def dumps(self, value): + return json.dumps(value, cls=DjangoJSONEncoder) + + def loads(self, value): + return json.loads(value) + + +class ConnectionFactory(Factory): + """ + Custom ConnectionFactory that injects the decode_responses parameter. + """ + + def make_connection_params(self, url): + kwargs = super().make_connection_params(url) + kwargs["decode_responses"] = True + return kwargs + diff --git a/backend/app/urls.py b/backend/app/urls.py index 933c809..3394ae1 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -1,8 +1,18 @@ + from django.urls import path -from app.views import BhavCopyEquityView, BhavCopyEquityDefaultRedisView, EmptyRespoinseView +from app.views import BhavCopyEquityView, BhavCopyEquityDefaultRedisView, BhavCopyEquityCustomRedisView, EmptyRespoinseView urlpatterns = [ path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'), path('bhavcopyequity/', BhavCopyEquityView.as_view(), name='bhavcopyequity_view'), - path('bhavcopyequitydefaultredis/', BhavCopyEquityDefaultRedisView.as_view(), name='bhavcopyequitydefaultredis_view'), + path( + 'bhavcopyequitydefaultredis/', + BhavCopyEquityDefaultRedisView.as_view(), + name='bhavcopyequitydefaultredis_view' + ), + path( + 'bhavcopyequitycustomredis/', + BhavCopyEquityCustomRedisView.as_view(), + name='bhavcopyequitycustomredis_view' + ), ] diff --git a/backend/app/utils.py b/backend/app/utils.py index 85166a8..148b947 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -11,16 +11,18 @@ from io import BytesIO, TextIOWrapper from zipfile import ZipFile # third-party imports -import django_rq import requests +from django_redis import get_redis_connection from django.db import transaction -from django_rq import job # app imports from app.models import BhavCopyEquity +cache = get_redis_connection("default") + + def fetch_bhav_copy_equity_data(curr_date=None): """Fetch data from BSE India website.""" # hack: since bseindia website doesn't respond well to python requests @@ -47,10 +49,34 @@ def fetch_bhav_copy_equity_data(curr_date=None): @transaction.atomic def populate_bhav_copy_data(): """Populate DB with Bhav Copy data.""" + pipe = cache.pipeline() data = fetch_bhav_copy_equity_data() - del data[0] + del data[0] # delete title row + + cache.delete("stocks") for stock in data: # prevent creation of duplicate entries + pipe.rpush("stocks", stock[0]) + pipe.hset( + f"stock:{stock[0]}", + mapping={ + "sc_code": stock[0], + "sc_name": stock[1], + "sc_group": stock[2], + "sc_type": stock[3], + "open_price": float(stock[4]), + "high_price": float(stock[5]), + "low_price": float(stock[6]), + "close_price": float(stock[7]), + "last_price": float(stock[8]), + "prevclose_price": float(stock[9]), + "no_trades": int(stock[10]), + "no_of_shrs": int(stock[11]), + "net_turnov": float(stock[12]), + "tdcloindi": stock[13], + } + ) + pipe.execute() BhavCopyEquity.objects.get_or_create( sc_code=int(stock[0]), sc_name=stock[1], diff --git a/backend/app/views.py b/backend/app/views.py index 9380f99..ad7ae7d 100644 --- a/backend/app/views.py +++ b/backend/app/views.py @@ -2,13 +2,17 @@ # third-party imports from rest_framework.response import Response from rest_framework import generics - +from django_redis import get_redis_connection from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_page # app imports from app.models import BhavCopyEquity from app.serializers import BhavCopyEquitySerializer +from app.utils import populate_bhav_copy_data + + +cache = get_redis_connection("default") # Create your views here. @@ -36,3 +40,14 @@ class EmptyRespoinseView(generics.RetrieveAPIView): return Response([]) +class BhavCopyEquityCustomRedisView(generics.RetrieveAPIView): + def get(self, request, *args, **kwargs): + pipe = cache.pipeline() + stocks = cache.lrange("stocks", 0, -1) + if len(stocks) == 0: + populate_bhav_copy_data() + stocks = cache.lrange("stocks", 0, -1) + for stock in stocks: + pipe.hgetall(f"stock:{stock}") + return Response(pipe.execute()) + diff --git a/backend/backend/settings.py b/backend/backend/settings.py index e79e194..7272d56 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -101,6 +101,10 @@ CACHES = { 'LOCATION': 'redis://redis:6379/1', 'OPTIONS': { 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + "REDIS_CLIENT_CLASS": "redis.client.StrictRedis", + "REDIS_CLIENT_KWARGS": {"decode_responses": True}, + # Custom serializer + "SERIALIZER": "app.redis.JSONSerializer", }, }, } @@ -155,6 +159,8 @@ RQ_QUEUES = { SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_CACHE_ALIAS = "default" CACHE_TTL = 60 * 5 +DJANGO_REDIS_CONNECTION_FACTORY = "app.redis.ConnectionFactory" + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ diff --git a/frontend/src/axios-api.js b/frontend/src/axios-api.js index 9b3ad55..b695be1 100644 --- a/frontend/src/axios-api.js +++ b/frontend/src/axios-api.js @@ -3,7 +3,7 @@ import axios from 'axios' // TODO: fix baseurl as per builds const getAPI = axios.create({ baseURL: 'http://127.0.0.1:8000', - timeout: 5000, + timeout: 60000, }) export { getAPI } diff --git a/frontend/src/views/BhavCopy.vue b/frontend/src/views/BhavCopy.vue index 64bdd7d..4cd45bd 100644 --- a/frontend/src/views/BhavCopy.vue +++ b/frontend/src/views/BhavCopy.vue @@ -96,11 +96,12 @@ return { sc_name: '', apiData: [], - apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Plain Endpoint" }, + apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Custom Redis Cache" }, apiEndpoints: [ { endpoint: 'emptyresponse', text: "Empty Response Endpoint" }, { endpoint: 'bhavcopyequity', text: "Plain Endpoint" }, - { endpoint: 'bhavcopyequitydefaultredis', text: "Default Redis Cache" } + { endpoint: 'bhavcopyequitydefaultredis', text: "Default Redis Cache" }, + { endpoint: 'bhavcopyequitycustomredis', text: "Custom Redis Cache" }, ], headersData: [ {text: 'Stock Code', value: 'sc_code'},