From 6cc1af314b864a41f14d3facf116dc98e44e489e Mon Sep 17 00:00:00 2001 From: Ameya Shenoy Date: Sat, 13 Feb 2021 08:21:55 +0530 Subject: [PATCH] refactor: better naming and restructure Signed-off-by: Ameya Shenoy --- backend/app/urls.py | 30 ++++++++-- backend/app/utils.py | 29 +++++++++- backend/app/views.py | 97 +++++++++++++++++++-------------- frontend/src/views/BhavCopy.vue | 10 ++-- 4 files changed, 113 insertions(+), 53 deletions(-) diff --git a/backend/app/urls.py b/backend/app/urls.py index 8f997b3..fd1347b 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -4,16 +4,38 @@ """Bullish project URLs.""" +# third-party imports from django.urls import path + +# app imports from app import views from app.views import EmptyRespoinseView + urlpatterns = [ - path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'), - path('bhavcopyequity/', views.bhavCopyEquityListPostgres, name='bhavcopyequity_view'), path( - 'bhavcopyequitycustomredis/', - views.bhavCopyEquityListRedis, + 'test/', + EmptyRespoinseView.as_view(), + name='test' + ), + path( + 'postgres/bhavcopy/equity/', + views.dbBhavCopyEquityList, + name='bhavcopyequity_view' + ), + path( + 'redis/bhavcopy/equity/v1/', + views.cacheBhavCopyEquityListV1, + name='bhavcopyequitycustomredis_view' + ), + path( + 'redis/bhavcopy/equity/v2/', + views.cacheBhavCopyEquityListV2, + name='bhavcopyequitycustomredis_view' + ), + path( + 'ephemeral/bhavcopy/equity/', + views.bseBhavCopyEquityList, name='bhavcopyequitycustomredis_view' ), ] diff --git a/backend/app/utils.py b/backend/app/utils.py index 9fab35d..c0133aa 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -5,6 +5,7 @@ # standard imports import csv +import json import datetime import logging @@ -64,11 +65,17 @@ def populate_bhav_copy_data(date=None): raise ValueError(f"Error fetching data from BSE for {date}\nDetails: {err}") else: del data[0] # delete title row - populate_bhav_copy_data_into_redis(data, date=date) populate_bhav_copy_data_into_postgres(data, date=date) + populate_bhav_copy_data_into_redis_v1(data, date=date) + populate_bhav_copy_data_into_redis_v2(data, date=date) -def populate_bhav_copy_data_into_redis(data, date): +def populate_bhav_copy_data_into_redis_v1(data=None, date=None): + if data is None: + data = fetch_bhav_copy_equity_data(date=date) + del data[0] + if date is None: + date = datetime.datetime.now().date() logger.info('Populating data into redis') datestr = date.strftime("%d%m%y") pipe = cache.pipeline() @@ -86,9 +93,25 @@ def populate_bhav_copy_data_into_redis(data, date): pipe.execute() +def populate_bhav_copy_data_into_redis_v2(data=None, date=None): + if data is None: + data = fetch_bhav_copy_equity_data(date=date) + del data[0] + if date is None: + date = datetime.datetime.now().date() + datestr = date.strftime("%d%m%y") + data = stocks_csv_to_json(data) + stocks_key = f"stocks:v2:{datestr}" + cache.delete(stocks_key) + cache.set(stocks_key, json.dumps(data)) + + @transaction.atomic -def populate_bhav_copy_data_into_postgres(data, date=None): +def populate_bhav_copy_data_into_postgres(data=None, date=None): logger.info('Populating data into postgres for %s', date) + if data is None: + data = fetch_bhav_copy_equity_data(date=date) + del data[0] if date is None: date = datetime.datetime.now().date() for stock in data: diff --git a/backend/app/views.py b/backend/app/views.py index 0b9e4e6..4d64767 100644 --- a/backend/app/views.py +++ b/backend/app/views.py @@ -4,7 +4,7 @@ """Bullish page views.""" # standard imports -import datetime +import json import logging # third-party imports @@ -22,7 +22,8 @@ from app.utils import ( fetch_bhav_copy_equity_data, stocks_csv_to_json, populate_bhav_copy_data_into_postgres, - populate_bhav_copy_data_into_redis, + populate_bhav_copy_data_into_redis_v1, + populate_bhav_copy_data_into_redis_v2, verify_date ) @@ -32,8 +33,13 @@ logger = logging.getLogger(__name__) # Create your views here. +class EmptyRespoinseView(generics.RetrieveAPIView): + def get(self, request, *args, **kwargs): + return Response([]) + + @api_view(['GET']) -def bhavCopyEquityListPostgres(request): +def dbBhavCopyEquityList(request): ret_message = "" req_date, ret_message = verify_date(request.query_params.get('date'), ret_message) @@ -42,61 +48,29 @@ def bhavCopyEquityListPostgres(request): # Fetch data if not present if len(serializer.data) == 0: - logger.info('Data not available in DB, trying to fetch from BSE') - try: - data = fetch_bhav_copy_equity_data(date=req_date) - except: - return Response({ - 'data': [], - 'message': "Unable to fetch data from BSE" - }) - del data[0] # delete title row logger.info('Enqueue background task to populate Postgres DB') - django_rq.enqueue(populate_bhav_copy_data_into_postgres, args=(data, req_date)) - logger.info('Return quick response') - stocks = stocks_csv_to_json(data) - return Response({ - "data": stocks, - "message": 'Data was directly sourced from BSE!' - }) + django_rq.enqueue(populate_bhav_copy_data_into_postgres, args=(None, req_date)) + logger.info('Data not available in Postgres, redirecting to BSE') + return _bseBhavCopyEquityList(request) return Response({ "data": serializer.data, "message": ret_message }) -class EmptyRespoinseView(generics.RetrieveAPIView): - def get(self, request, *args, **kwargs): - return Response([]) - - @api_view(['GET']) -def bhavCopyEquityListRedis(request): +def cacheBhavCopyEquityListV1(request): ret_message = "" date = request.query_params.get('date') req_date, ret_message = verify_date(date, ret_message) datestr = req_date.strftime("%d%m%y") - # TODO: try stringified json stocks = cache.lrange(f"stocks:{datestr}", 0, -1) if len(stocks) == 0: - logger.info('Data not available in Redis, trying to fetch from BSE') - try: - data = fetch_bhav_copy_equity_data(date=req_date) - except: - return Response({ - 'data': [], - 'message': "Unable to fetch data from BSE" - }) - del data[0] # delete title row logger.info('Enqueue background task to populate Redis') - django_rq.enqueue(populate_bhav_copy_data_into_redis, args=(data, req_date)) - logger.info('Return quick response') - stocks = stocks_csv_to_json(data) - return Response({ - "data": stocks, - "message": 'Data was directly sourced from BSE!' - }) + django_rq.enqueue(populate_bhav_copy_data_into_redis_v1, args=(None, req_date)) + logger.info('Data not available in Redis, trying to fetch from BSE') + return _bseBhavCopyEquityList(request) pipe = cache.pipeline() for stock in stocks: pipe.hgetall(f"stock:{datestr}:{stock}") @@ -106,4 +80,43 @@ def bhavCopyEquityListRedis(request): }) +@api_view(['GET']) +def cacheBhavCopyEquityListV2(request): + ret_message = "" + date = request.query_params.get('date') + req_date, ret_message = verify_date(date, ret_message) + datestr = req_date.strftime("%d%m%y") + + stocks = cache.get(f"stocks:v2:{datestr}") + if stocks is None: + logger.info('Enqueue background task to populate Redis') + django_rq.enqueue(populate_bhav_copy_data_into_redis_v2, args=(None, req_date)) + logger.info('Data not available in Redis, trying to fetch from BSE') + return _bseBhavCopyEquityList(request) + return Response({ + "data": json.loads(stocks), + "message": ret_message + }) + + +@api_view(['GET']) +def bseBhavCopyEquityList(request): + return _bseBhavCopyEquityList(request) + +def _bseBhavCopyEquityList(request): + date = request.query_params.get('date') + req_date, _ = verify_date(date, "") + try: + data = fetch_bhav_copy_equity_data(date=req_date) + except: + return Response({ + 'data': [], + 'message': "Unable to fetch data from BSE" + }) + del data[0] # delete title row + stocks = stocks_csv_to_json(data) + return Response({ + "data": stocks, + "message": 'Data was directly sourced from BSE!' + }) diff --git a/frontend/src/views/BhavCopy.vue b/frontend/src/views/BhavCopy.vue index 61dbcd8..1cf8a5c 100644 --- a/frontend/src/views/BhavCopy.vue +++ b/frontend/src/views/BhavCopy.vue @@ -193,11 +193,13 @@ snackbarText: '', snackbarColor: '', apiData: [], - apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Redis Cache" }, + apiEndpointSelected: { endpoint: 'redis/bhavcopy/equity/v2', text: "Redis Cache v2" }, apiEndpoints: [ - { endpoint: 'emptyresponse', text: "Empty Response Endpoint" }, - { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" }, - { endpoint: 'bhavcopyequitycustomredis', text: "Redis Cache" }, + { endpoint: 'test', text: "Empty Response Endpoint" }, + { endpoint: 'ephemeral/bhavcopy/equity', text: "Directly from BSE" }, + { endpoint: 'postgres/bhavcopy/equity', text: "Postgres Endpoint" }, + { endpoint: 'redis/bhavcopy/equity/v1', text: "Redis Cache" }, + { endpoint: 'redis/bhavcopy/equity/v2', text: "Redis Cache v2" }, ], headersData: [ {text: 'Stock Code', value: 'sc_code'},