refactor: rewrite views for optimization

Signed-off-by: Ameya Shenoy <shenoy.ameya@gmail.com>

backend/app/urls.py

@@ -6,17 +6,14 @@
 
 from django.urls import path
 from app import views
-from app.views import (
+from app.views import EmptyRespoinseView
-    EmptyRespoinseView,
-    BhavCopyEquityCustomRedisView,
-)
 
 urlpatterns = [
     path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'),
-    path('bhavcopyequity/', views.bhavCopyEquityList, name='bhavcopyequity_view'),
+    path('bhavcopyequity/', views.bhavCopyEquityListPostgres, name='bhavcopyequity_view'),
     path(
         'bhavcopyequitycustomredis/',
-        BhavCopyEquityCustomRedisView.as_view(),
+        views.bhavCopyEquityListRedis,
         name='bhavcopyequitycustomredis_view'
     ),
 ]

backend/app/utils.py

@@ -64,35 +64,24 @@ def populate_bhav_copy_data(date=None):
         raise ValueError(f"Error fetching data from BSE for {date}\nDetails: {err}")
     else:
         del data[0]  # delete title row
-        populate_bhav_copy_data_into_redis(data)
+        populate_bhav_copy_data_into_redis(data, date=date)
         populate_bhav_copy_data_into_postgres(data, date=date)
 
 
-def populate_bhav_copy_data_into_redis(data):
+def populate_bhav_copy_data_into_redis(data, date):
     logger.info('Populating data into redis')
+    datestr = date.strftime("%d%m%y")
     pipe = cache.pipeline()
-    cache.delete("stocks")
+    stocks_key = f"stocks:{datestr}"
+    cache.delete(stocks_key)
+    data = stocks_csv_to_json(data)
     for stock in data:
         # prevent creation of duplicate entries
-        pipe.rpush("stocks", stock[0])
+        stock_code = stock.get('sc_code')
+        pipe.rpush(stocks_key, stock_code)
         pipe.hset(
-            f"stock:{stock[0]}",
+            f"stock:{datestr}:{stock_code}",
-            mapping={
+            mapping=stock
-                "sc_code": stock[0],
-                "sc_name": stock[1],
-                "sc_group": stock[2],
-                "sc_type": stock[3],
-                "open_price": float(stock[4]),
-                "high_price": float(stock[5]),
-                "low_price": float(stock[6]),
-                "close_price": float(stock[7]),
-                "last_price": float(stock[8]),
-                "prevclose_price": float(stock[9]),
-                "no_trades": int(stock[10]),
-                "no_of_shrs": int(stock[11]),
-                "net_turnov": float(stock[12]),
-                "tdcloindi": stock[13],
-            }
         )
     pipe.execute()
 
@@ -143,3 +132,18 @@ def stocks_csv_to_json(data):
         })
     return stocks
 
+
+def verify_date(date, ret_message):
+    # Verify Date
+    logger.info('Verifying date %s', date)
+    req_date = datetime.datetime.strptime(date, '%Y-%m-%d')
+    # 18:00 IST == 12:30 UTC
+    today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0)
+    logger.info('Req Date: %s; Today: %s', req_date, today)
+    if req_date > today:
+        ret_message = "Time travel not yet invented! Returning latest available data."
+        req_date = today
+    if req_date.date() == today.date() and req_date < today:
+        req_date = today - datetime.timedelta(days=1)
+    return req_date, ret_message
+

backend/app/views.py

@@ -19,10 +19,11 @@ from django_redis import get_redis_connection
 from app.models import BhavCopyEquity
 from app.serializers import BhavCopyEquitySerializer
 from app.utils import (
-    populate_bhav_copy_data,
     fetch_bhav_copy_equity_data,
     stocks_csv_to_json,
-    populate_bhav_copy_data_into_postgres
+    populate_bhav_copy_data_into_postgres,
+    populate_bhav_copy_data_into_redis,
+    verify_date
 )
 
 
@@ -32,18 +33,10 @@ logger = logging.getLogger(__name__)
 
 # Create your views here.
 @api_view(['GET'])
-def bhavCopyEquityList(request):
+def bhavCopyEquityListPostgres(request):
     ret_message = ""
+    req_date, ret_message = verify_date(request.query_params.get('date'), ret_message)
 
-    # Verify Date
-    req_date = datetime.datetime.strptime(request.query_params.get('date'), '%Y-%m-%d')
-    # 18:00 IST == 12:30 UTC
-    today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0)
-    if req_date > today:
-        ret_message = "Time travel not yet invented! Returning latest available data."
-        req_date = today
-        if datetime.datetime.now() < today:
-            req_date = today - datetime.timedelta(days=1)
     queryset = BhavCopyEquity.objects.all().filter(date=req_date)
     serializer = BhavCopyEquitySerializer(queryset, many=True)
 
@@ -77,15 +70,40 @@ class EmptyRespoinseView(generics.RetrieveAPIView):
         return Response([])
 
 
-class BhavCopyEquityCustomRedisView(generics.RetrieveAPIView):
+@api_view(['GET'])
-    def get(self, request, *args, **kwargs):
+def bhavCopyEquityListRedis(request):
-        # TODO: try stringified json
+    ret_message = ""
-        pipe = cache.pipeline()
+    date = request.query_params.get('date')
-        stocks = cache.lrange("stocks", 0, -1)
+    req_date, ret_message = verify_date(date, ret_message)
-        if len(stocks) == 0:
+    datestr = req_date.strftime("%d%m%y")
-            populate_bhav_copy_data()
+
-            stocks = cache.lrange("stocks", 0, -1)
+    # TODO: try stringified json
-        for stock in stocks:
+    stocks = cache.lrange(f"stocks:{datestr}", 0, -1)
-            pipe.hgetall(f"stock:{stock}")
+    if len(stocks) == 0:
-        return Response(pipe.execute())
+        logger.info('Data not available in Redis, trying to fetch from BSE')
+        try:
+            data = fetch_bhav_copy_equity_data(date=req_date)
+        except:
+            return Response({
+                'data': [],
+                'message': "Unable to fetch data from BSE"
+            })
+        del data[0]  # delete title row
+        logger.info('Enqueue background task to populate Redis')
+        django_rq.enqueue(populate_bhav_copy_data_into_redis, args=(data, req_date))
+        logger.info('Return quick response')
+        stocks = stocks_csv_to_json(data)
+        return Response({
+            "data": stocks,
+            "message": 'Data was directly sourced from BSE!'
+        })
+    pipe = cache.pipeline()
+    for stock in stocks:
+        pipe.hgetall(f"stock:{datestr}:{stock}")
+    return Response({
+        "data": pipe.execute(),
+        "message": ret_message
+    })
+
+
 

frontend/src/views/BhavCopy.vue

@@ -193,7 +193,7 @@
         snackbarText: '',
         snackbarColor: '',
         apiData: [],
-        apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },
+        apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Redis Cache" },
         apiEndpoints: [
           { endpoint: 'emptyresponse', text: "Empty Response Endpoint" },
           { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },