feat: implement date picker

- re write postgres endpoint
- enable logging in django
- ui for date picker

Signed-off-by: Ameya Shenoy <shenoy.ameya@gmail.com>

README.md

@@ -59,6 +59,28 @@ docker logs -f container-name
 Here `container-name` can be either of `backend`, `frontend`, `redis`,
 `postgresql`, `rqworker` or `rqscheduler`.
 
+### Debugging
+
+- To goto Django's shell to debug you may use
+
+```sh
+docker-compose exec backend python manage.py shell
+```
+
+- To create migrations
+
+```sh
+docker-compose exec backend python manage.py makemigrations
+```
+
+- To run migrate
+
+```sh
+docker-compose exec backend python manage.py migrate
+```
+
+Similarly all django commands can be run this way.
+
 
 ## Deploy
 

backend/app/migrations/0003_auto_20210212_1348.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.6 on 2021-02-12 13:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0002_auto_20210209_0920'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bhavcopyequity',
+            name='date',
+            field=models.DateField(editable=False),
+        ),
+    ]

backend/app/migrations/0004_auto_20210212_1846.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.6 on 2021-02-12 18:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0003_auto_20210212_1348'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bhavcopyequity',
+            name='date',
+            field=models.DateField(),
+        ),
+    ]

backend/app/models.py

@@ -4,7 +4,7 @@ from django.db import models
 
 class BhavCopyEquity(models.Model):
     id = models.AutoField(primary_key=True)
-    date = models.DateField(auto_now_add=True, editable=False)
+    date = models.DateField()
 
     # Ref: https://www.bseindia.com/markets/debt/BhavCopyhelp.aspx
     sc_code = models.PositiveIntegerField()

backend/app/urls.py

@@ -5,15 +5,15 @@
 
 
 from django.urls import path
+from app import views
 from app.views import (
-    BhavCopyEquityView,
+    EmptyRespoinseView,
     BhavCopyEquityCustomRedisView,
-    EmptyRespoinseView
 )
 
 urlpatterns = [
     path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'),
-    path('bhavcopyequity/', BhavCopyEquityView.as_view(), name='bhavcopyequity_view'),
+    path('bhavcopyequity/', views.bhavCopyEquityList, name='bhavcopyequity_view'),
     path(
         'bhavcopyequitycustomredis/',
         BhavCopyEquityCustomRedisView.as_view(),

backend/app/utils.py

@@ -6,6 +6,7 @@
 # standard imports
 import csv
 import datetime
+import logging
 
 from io import BytesIO, TextIOWrapper
 from zipfile import ZipFile
@@ -22,18 +23,20 @@ from app.models import BhavCopyEquity
 
 
 cache = get_redis_connection("default")
+logger = logging.getLogger(__name__)
 
 
-def fetch_bhav_copy_equity_data(curr_date=None):
+def fetch_bhav_copy_equity_data(date=None):
     """Fetch data from BSE India website."""
     # hack: since bseindia website doesn't respond well to python requests
     user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0'
     headers = {
         'User-Agent': user_agent
     }
-    if curr_date is None:
+    if date is None:
-        curr_date = datetime.datetime.now().strftime("%d%m%y")
+        date = datetime.datetime.now()
-    zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP'
+    zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{date.strftime("%d%m%y")}_CSV.ZIP'
+    logger.info(f'Fetching data from {zipurl}')
     resp = requests.get(zipurl, headers=headers)
 
     if resp.ok:
@@ -42,57 +45,13 @@ def fetch_bhav_copy_equity_data(curr_date=None):
             with bhavcopy_zf.open(csv_file, 'r') as infile:
                 data = list(csv.reader(TextIOWrapper(infile, 'utf-8')))
         return data
-    return None
+    raise ValueError('Fetching data from BSE unsuccessful')
 
 
-@transaction.atomic
+def populate_bhav_copy_data(date=None):
-def populate_bhav_copy_data():
     """Populate DB with Bhav Copy data."""
     try:
-        pipe = cache.pipeline()
+        data = fetch_bhav_copy_equity_data(date=date)
-        data = fetch_bhav_copy_equity_data()
-        del data[0]  # delete title row
-
-        cache.delete("stocks")
-        for stock in data:
-            # prevent creation of duplicate entries
-            pipe.rpush("stocks", stock[0])
-            pipe.hset(
-                f"stock:{stock[0]}",
-                mapping={
-                    "sc_code": stock[0],
-                    "sc_name": stock[1],
-                    "sc_group": stock[2],
-                    "sc_type": stock[3],
-                    "open_price": float(stock[4]),
-                    "high_price": float(stock[5]),
-                    "low_price": float(stock[6]),
-                    "close_price": float(stock[7]),
-                    "last_price": float(stock[8]),
-                    "prevclose_price": float(stock[9]),
-                    "no_trades": int(stock[10]),
-                    "no_of_shrs": int(stock[11]),
-                    "net_turnov": float(stock[12]),
-                    "tdcloindi": stock[13],
-                }
-            )
-            pipe.execute()
-            BhavCopyEquity.objects.get_or_create(
-                sc_code=int(stock[0]),
-                sc_name=stock[1],
-                sc_group=stock[2],
-                sc_type=stock[3],
-                open_price=float(stock[4]),
-                high_price=float(stock[5]),
-                low_price=float(stock[6]),
-                close_price=float(stock[7]),
-                last_price=float(stock[8]),
-                prevclose_price=float(stock[9]),
-                no_trades=int(stock[10]),
-                no_of_shrs=int(stock[11]),
-                net_turnov=float(stock[12]),
-                tdcloindi=stock[13],
-            )
     except:
         # Potentially add code for alerting if needed
         # Repeat job after 10 mins if job fails
@@ -101,4 +60,63 @@ def populate_bhav_copy_data():
             scheduled_time=datetime.datetime.now()+datetime.timedelta(minutes=10),
             func=populate_bhav_copy_data,
         )
+        raise ValueError(f"Error fetching data from BSE for {date}")
+    else:
+        del data[0]  # delete title row
+        populate_bhav_copy_data_into_redis(data)
+        populate_bhav_copy_data_into_postgres(data, date=date)
+
+
+def populate_bhav_copy_data_into_redis(data):
+    logger.info(f'Populating data into redis')
+    pipe = cache.pipeline()
+    cache.delete("stocks")
+    for stock in data:
+        # prevent creation of duplicate entries
+        pipe.rpush("stocks", stock[0])
+        pipe.hset(
+            f"stock:{stock[0]}",
+            mapping={
+                "sc_code": stock[0],
+                "sc_name": stock[1],
+                "sc_group": stock[2],
+                "sc_type": stock[3],
+                "open_price": float(stock[4]),
+                "high_price": float(stock[5]),
+                "low_price": float(stock[6]),
+                "close_price": float(stock[7]),
+                "last_price": float(stock[8]),
+                "prevclose_price": float(stock[9]),
+                "no_trades": int(stock[10]),
+                "no_of_shrs": int(stock[11]),
+                "net_turnov": float(stock[12]),
+                "tdcloindi": stock[13],
+            }
+        )
+    pipe.execute()
+
+
+@transaction.atomic
+def populate_bhav_copy_data_into_postgres(data, date=None):
+    logger.info(f'Populating data into postgres for {date}')
+    if date is None:
+        date = datetime.datetime.now().date()
+    for stock in data:
+        BhavCopyEquity.objects.get_or_create(
+            date=date,
+            sc_code=int(stock[0]),
+            sc_name=stock[1],
+            sc_group=stock[2],
+            sc_type=stock[3],
+            open_price=float(stock[4]),
+            high_price=float(stock[5]),
+            low_price=float(stock[6]),
+            close_price=float(stock[7]),
+            last_price=float(stock[8]),
+            prevclose_price=float(stock[9]),
+            no_trades=int(stock[10]),
+            no_of_shrs=int(stock[11]),
+            net_turnov=float(stock[12]),
+            tdcloindi=stock[13],
+        )
 

backend/app/views.py

@@ -3,13 +3,15 @@
 
 """Bullish page views."""
 
+# standard imports
+import datetime
+import logging
 
 # third-party imports
-from rest_framework.response import Response
 from rest_framework import generics
+from rest_framework.response import Response
+from rest_framework.decorators import api_view
 from django_redis import get_redis_connection
-from django.utils.decorators import method_decorator
-from django.views.decorators.cache import cache_page
 
 # app imports
 from app.models import BhavCopyEquity
@@ -18,16 +20,37 @@ from app.utils import populate_bhav_copy_data
 
 
 cache = get_redis_connection("default")
+logger = logging.getLogger(__name__)
 
 
 # Create your views here.
-class BhavCopyEquityView(generics.RetrieveAPIView):
+@api_view(['GET'])
-    queryset = BhavCopyEquity.objects.all()
+def bhavCopyEquityList(request):
+    ret_message = ""
 
-    def get(self, request, *args, **kwargs):
+    # Verify Date
-        queryset = self.get_queryset()
+    req_date = datetime.datetime.strptime(request.query_params.get('date'), '%Y-%m-%d')
+    # 18:00 IST == 12:30 UTC
+    today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0)
+    if req_date > today:
+        ret_message = "Time travel not yet invented! Returning latest available data."
+        req_date = today
+        if datetime.datetime.now() < today:
+            req_date = today - datetime.timedelta(days=1)
+    queryset = BhavCopyEquity.objects.all().filter(date=req_date)
+    serializer = BhavCopyEquitySerializer(queryset, many=True)
+
+    # Fetch data if not present
+    if len(serializer.data) == 0:
+        logger.info(f'Data not available in DB')
+        populate_bhav_copy_data(date=req_date)
+        queryset = BhavCopyEquity.objects.all().filter(date=req_date)
         serializer = BhavCopyEquitySerializer(queryset, many=True)
-        return Response(serializer.data)
+
+    return Response({
+        "data": serializer.data,
+        "message": ret_message
+    })
 
 
 class EmptyRespoinseView(generics.RetrieveAPIView):

backend/backend/settings.py

@@ -163,6 +163,20 @@ SESSION_CACHE_ALIAS = "default"
 CACHE_TTL = 60 * 5
 DJANGO_REDIS_CONNECTION_FACTORY = "app.redis.ConnectionFactory"
 
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': 'INFO',
+    },
+}
+
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/

frontend/src/views/BhavCopy.vue

@@ -23,19 +23,36 @@
     <v-card-title>
       Bhav Copy - Equity
       <v-spacer></v-spacer>
-        <v-btn
+      <v-menu
-          class="ma-2"
+        ref="datePickerMenu"
-          color="blue lighten-2"
+        v-model="datePickerMenu"
-          @click="downloadcsv"
+        :close-on-content-click="false"
-        >
+        transition="scale-transition"
-          Download
+        offset-y
-          <v-icon
+        max-width="290px"
-            light
+        min-width="auto"
-            right
+      >
-          >
+        <template v-slot:activator="{ on, attrs }">
-            mdi-cloud-download
+          <v-text-field
-          </v-icon>
+            v-model="dateFormatted"
-        </v-btn>
+            label="Date"
+            hint="MM/DD/YYYY format"
+            persistent-hint
+            prepend-icon="mdi-calendar"
+            v-bind="attrs"
+            @blur="date = parseDate(dateFormatted)"
+            v-on="on"
+          ></v-text-field>
+        </template>
+        <v-date-picker
+          v-model="date"
+          no-title
+          v-on:change="populateApiData"
+          @input="datePickerMenu = false"
+          min="2007-01-02"
+          v-bind:max="(new Date()).toISOString().slice(0, 10)"
+        ></v-date-picker>
+      </v-menu>
       <v-spacer></v-spacer>
       <v-select
         v-model="apiEndpointSelected.endpoint"
@@ -45,6 +62,20 @@
         item-value="endpoint"
       ></v-select>
       <v-spacer></v-spacer>
+      <v-btn
+        class="ma-2"
+        color="blue lighten-2"
+        @click="downloadcsv"
+      >
+        Download
+        <v-icon
+          light
+          right
+        >
+          mdi-cloud-download
+        </v-icon>
+      </v-btn>
+      <v-spacer></v-spacer>
       <v-text-field
         ref="searchField"
         v-model="sc_name"
@@ -100,36 +131,58 @@
       },
       populateApiData() {
         var endpoint = this.apiEndpointSelected.endpoint;
-        console.log(`Fetching data from ${endpoint} API`)
+        console.log(`Fetching data from ${endpoint} API for ${this.date} date`)
-        getAPI.get(`/${endpoint}/`,)
+        getAPI.get(`/${endpoint}/`, {
+            params: {
+              date: this.date
+            }
+          })
           .then(response => {
             console.log('BhavCopyEquity API has recieved data')
-            this.apiData = response.data
+            this.apiData = response.data.data
-            this.snackbarText = "Successfully loaded data"
+            this.snackbarText = response.data.message || "Successfully loaded data"
-            this.snackbarColor = "green lighten-2"
+            this.snackbarColor = `${response.data.message ? 'yellow' : 'green'} lighten-2`
             this.snackbar = true
           })
           .catch(err => {
             console.log(err)
-            this.snackbarText = "Error fetching data"
+            this.snackbarText = "Server Error"
             this.snackbarColor = "red lighten-2"
             this.snackbar = true
           })
       },
+      formatDate (date) {
+        if (!date) return null
+
+        const [year, month, day] = date.split('-')
+        return `${month}/${day}/${year}`
+      },
+      parseDate (date) {
+        if (!date) return null
+
+        const [month, day, year] = date.split('/')
+        return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`
+      },
     },
     computed: {
       headers() {
         return this.headersData
-      }
+      },
+      computedDateFormatted () {
+        return this.formatDate(this.date)
+      },
     },
     data() {
       return {
         sc_name: '',
+        date: new Date().toISOString().substr(0, 10),
+        dateFormatted: '',
+        datePickerMenu: false,
         snackbar: false,
         snackbarText: '',
         snackbarColor: '',
         apiData: [],
-        apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Custom Redis Cache" },
+        apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },
         apiEndpoints: [
           { endpoint: 'emptyresponse', text: "Empty Response Endpoint" },
           { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },
@@ -161,7 +214,12 @@
     },
     created() {
       this.populateApiData()
-    }
+    },
+    watch: {
+      date () {
+        this.dateFormatted = this.formatDate(this.date)
+      },
+    },
   }
 </script>