diff --git a/backend/app/urls.py b/backend/app/urls.py index cc703c4..8f997b3 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -6,17 +6,14 @@ from django.urls import path from app import views -from app.views import ( - EmptyRespoinseView, - BhavCopyEquityCustomRedisView, -) +from app.views import EmptyRespoinseView urlpatterns = [ path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'), - path('bhavcopyequity/', views.bhavCopyEquityList, name='bhavcopyequity_view'), + path('bhavcopyequity/', views.bhavCopyEquityListPostgres, name='bhavcopyequity_view'), path( 'bhavcopyequitycustomredis/', - BhavCopyEquityCustomRedisView.as_view(), + views.bhavCopyEquityListRedis, name='bhavcopyequitycustomredis_view' ), ] diff --git a/backend/app/utils.py b/backend/app/utils.py index e417832..9fab35d 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -64,35 +64,24 @@ def populate_bhav_copy_data(date=None): raise ValueError(f"Error fetching data from BSE for {date}\nDetails: {err}") else: del data[0] # delete title row - populate_bhav_copy_data_into_redis(data) + populate_bhav_copy_data_into_redis(data, date=date) populate_bhav_copy_data_into_postgres(data, date=date) -def populate_bhav_copy_data_into_redis(data): +def populate_bhav_copy_data_into_redis(data, date): logger.info('Populating data into redis') + datestr = date.strftime("%d%m%y") pipe = cache.pipeline() - cache.delete("stocks") + stocks_key = f"stocks:{datestr}" + cache.delete(stocks_key) + data = stocks_csv_to_json(data) for stock in data: # prevent creation of duplicate entries - pipe.rpush("stocks", stock[0]) + stock_code = stock.get('sc_code') + pipe.rpush(stocks_key, stock_code) pipe.hset( - f"stock:{stock[0]}", - mapping={ - "sc_code": stock[0], - "sc_name": stock[1], - "sc_group": stock[2], - "sc_type": stock[3], - "open_price": float(stock[4]), - "high_price": float(stock[5]), - "low_price": float(stock[6]), - "close_price": float(stock[7]), - "last_price": float(stock[8]), - "prevclose_price": float(stock[9]), - "no_trades": int(stock[10]), - "no_of_shrs": int(stock[11]), - "net_turnov": float(stock[12]), - "tdcloindi": stock[13], - } + f"stock:{datestr}:{stock_code}", + mapping=stock ) pipe.execute() @@ -143,3 +132,18 @@ def stocks_csv_to_json(data): }) return stocks + +def verify_date(date, ret_message): + # Verify Date + logger.info('Verifying date %s', date) + req_date = datetime.datetime.strptime(date, '%Y-%m-%d') + # 18:00 IST == 12:30 UTC + today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0) + logger.info('Req Date: %s; Today: %s', req_date, today) + if req_date > today: + ret_message = "Time travel not yet invented! Returning latest available data." + req_date = today + if req_date.date() == today.date() and req_date < today: + req_date = today - datetime.timedelta(days=1) + return req_date, ret_message + diff --git a/backend/app/views.py b/backend/app/views.py index 7ddc444..0b9e4e6 100644 --- a/backend/app/views.py +++ b/backend/app/views.py @@ -19,10 +19,11 @@ from django_redis import get_redis_connection from app.models import BhavCopyEquity from app.serializers import BhavCopyEquitySerializer from app.utils import ( - populate_bhav_copy_data, fetch_bhav_copy_equity_data, stocks_csv_to_json, - populate_bhav_copy_data_into_postgres + populate_bhav_copy_data_into_postgres, + populate_bhav_copy_data_into_redis, + verify_date ) @@ -32,18 +33,10 @@ logger = logging.getLogger(__name__) # Create your views here. @api_view(['GET']) -def bhavCopyEquityList(request): +def bhavCopyEquityListPostgres(request): ret_message = "" + req_date, ret_message = verify_date(request.query_params.get('date'), ret_message) - # Verify Date - req_date = datetime.datetime.strptime(request.query_params.get('date'), '%Y-%m-%d') - # 18:00 IST == 12:30 UTC - today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0) - if req_date > today: - ret_message = "Time travel not yet invented! Returning latest available data." - req_date = today - if datetime.datetime.now() < today: - req_date = today - datetime.timedelta(days=1) queryset = BhavCopyEquity.objects.all().filter(date=req_date) serializer = BhavCopyEquitySerializer(queryset, many=True) @@ -77,15 +70,40 @@ class EmptyRespoinseView(generics.RetrieveAPIView): return Response([]) -class BhavCopyEquityCustomRedisView(generics.RetrieveAPIView): - def get(self, request, *args, **kwargs): - # TODO: try stringified json - pipe = cache.pipeline() - stocks = cache.lrange("stocks", 0, -1) - if len(stocks) == 0: - populate_bhav_copy_data() - stocks = cache.lrange("stocks", 0, -1) - for stock in stocks: - pipe.hgetall(f"stock:{stock}") - return Response(pipe.execute()) +@api_view(['GET']) +def bhavCopyEquityListRedis(request): + ret_message = "" + date = request.query_params.get('date') + req_date, ret_message = verify_date(date, ret_message) + datestr = req_date.strftime("%d%m%y") + + # TODO: try stringified json + stocks = cache.lrange(f"stocks:{datestr}", 0, -1) + if len(stocks) == 0: + logger.info('Data not available in Redis, trying to fetch from BSE') + try: + data = fetch_bhav_copy_equity_data(date=req_date) + except: + return Response({ + 'data': [], + 'message': "Unable to fetch data from BSE" + }) + del data[0] # delete title row + logger.info('Enqueue background task to populate Redis') + django_rq.enqueue(populate_bhav_copy_data_into_redis, args=(data, req_date)) + logger.info('Return quick response') + stocks = stocks_csv_to_json(data) + return Response({ + "data": stocks, + "message": 'Data was directly sourced from BSE!' + }) + pipe = cache.pipeline() + for stock in stocks: + pipe.hgetall(f"stock:{datestr}:{stock}") + return Response({ + "data": pipe.execute(), + "message": ret_message + }) + + diff --git a/frontend/src/views/BhavCopy.vue b/frontend/src/views/BhavCopy.vue index aebfaea..61dbcd8 100644 --- a/frontend/src/views/BhavCopy.vue +++ b/frontend/src/views/BhavCopy.vue @@ -193,7 +193,7 @@ snackbarText: '', snackbarColor: '', apiData: [], - apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" }, + apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Redis Cache" }, apiEndpoints: [ { endpoint: 'emptyresponse', text: "Empty Response Endpoint" }, { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },