diff --git a/README.md b/README.md index f7cf2ad..457a1ef 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,28 @@ docker logs -f container-name Here `container-name` can be either of `backend`, `frontend`, `redis`, `postgresql`, `rqworker` or `rqscheduler`. +### Debugging + +- To goto Django's shell to debug you may use + +```sh +docker-compose exec backend python manage.py shell +``` + +- To create migrations + +```sh +docker-compose exec backend python manage.py makemigrations +``` + +- To run migrate + +```sh +docker-compose exec backend python manage.py migrate +``` + +Similarly all django commands can be run this way. + ## Deploy diff --git a/backend/app/migrations/0003_auto_20210212_1348.py b/backend/app/migrations/0003_auto_20210212_1348.py new file mode 100644 index 0000000..a113c51 --- /dev/null +++ b/backend/app/migrations/0003_auto_20210212_1348.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.6 on 2021-02-12 13:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0002_auto_20210209_0920'), + ] + + operations = [ + migrations.AlterField( + model_name='bhavcopyequity', + name='date', + field=models.DateField(editable=False), + ), + ] diff --git a/backend/app/migrations/0004_auto_20210212_1846.py b/backend/app/migrations/0004_auto_20210212_1846.py new file mode 100644 index 0000000..464b3b3 --- /dev/null +++ b/backend/app/migrations/0004_auto_20210212_1846.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.6 on 2021-02-12 18:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0003_auto_20210212_1348'), + ] + + operations = [ + migrations.AlterField( + model_name='bhavcopyequity', + name='date', + field=models.DateField(), + ), + ] diff --git a/backend/app/models.py b/backend/app/models.py index 53fc5f8..8af5785 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -4,7 +4,7 @@ from django.db import models class BhavCopyEquity(models.Model): id = models.AutoField(primary_key=True) - date = models.DateField(auto_now_add=True, editable=False) + date = models.DateField() # Ref: https://www.bseindia.com/markets/debt/BhavCopyhelp.aspx sc_code = models.PositiveIntegerField() diff --git a/backend/app/urls.py b/backend/app/urls.py index 23bd3a4..cc703c4 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -5,15 +5,15 @@ from django.urls import path +from app import views from app.views import ( - BhavCopyEquityView, + EmptyRespoinseView, BhavCopyEquityCustomRedisView, - EmptyRespoinseView ) urlpatterns = [ path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'), - path('bhavcopyequity/', BhavCopyEquityView.as_view(), name='bhavcopyequity_view'), + path('bhavcopyequity/', views.bhavCopyEquityList, name='bhavcopyequity_view'), path( 'bhavcopyequitycustomredis/', BhavCopyEquityCustomRedisView.as_view(), diff --git a/backend/app/utils.py b/backend/app/utils.py index c514678..0b87ce3 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -6,6 +6,7 @@ # standard imports import csv import datetime +import logging from io import BytesIO, TextIOWrapper from zipfile import ZipFile @@ -22,18 +23,20 @@ from app.models import BhavCopyEquity cache = get_redis_connection("default") +logger = logging.getLogger(__name__) -def fetch_bhav_copy_equity_data(curr_date=None): +def fetch_bhav_copy_equity_data(date=None): """Fetch data from BSE India website.""" # hack: since bseindia website doesn't respond well to python requests user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0' headers = { 'User-Agent': user_agent } - if curr_date is None: - curr_date = datetime.datetime.now().strftime("%d%m%y") - zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP' + if date is None: + date = datetime.datetime.now() + zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{date.strftime("%d%m%y")}_CSV.ZIP' + logger.info(f'Fetching data from {zipurl}') resp = requests.get(zipurl, headers=headers) if resp.ok: @@ -42,57 +45,13 @@ def fetch_bhav_copy_equity_data(curr_date=None): with bhavcopy_zf.open(csv_file, 'r') as infile: data = list(csv.reader(TextIOWrapper(infile, 'utf-8'))) return data - return None + raise ValueError('Fetching data from BSE unsuccessful') -@transaction.atomic -def populate_bhav_copy_data(): +def populate_bhav_copy_data(date=None): """Populate DB with Bhav Copy data.""" try: - pipe = cache.pipeline() - data = fetch_bhav_copy_equity_data() - del data[0] # delete title row - - cache.delete("stocks") - for stock in data: - # prevent creation of duplicate entries - pipe.rpush("stocks", stock[0]) - pipe.hset( - f"stock:{stock[0]}", - mapping={ - "sc_code": stock[0], - "sc_name": stock[1], - "sc_group": stock[2], - "sc_type": stock[3], - "open_price": float(stock[4]), - "high_price": float(stock[5]), - "low_price": float(stock[6]), - "close_price": float(stock[7]), - "last_price": float(stock[8]), - "prevclose_price": float(stock[9]), - "no_trades": int(stock[10]), - "no_of_shrs": int(stock[11]), - "net_turnov": float(stock[12]), - "tdcloindi": stock[13], - } - ) - pipe.execute() - BhavCopyEquity.objects.get_or_create( - sc_code=int(stock[0]), - sc_name=stock[1], - sc_group=stock[2], - sc_type=stock[3], - open_price=float(stock[4]), - high_price=float(stock[5]), - low_price=float(stock[6]), - close_price=float(stock[7]), - last_price=float(stock[8]), - prevclose_price=float(stock[9]), - no_trades=int(stock[10]), - no_of_shrs=int(stock[11]), - net_turnov=float(stock[12]), - tdcloindi=stock[13], - ) + data = fetch_bhav_copy_equity_data(date=date) except: # Potentially add code for alerting if needed # Repeat job after 10 mins if job fails @@ -101,4 +60,63 @@ def populate_bhav_copy_data(): scheduled_time=datetime.datetime.now()+datetime.timedelta(minutes=10), func=populate_bhav_copy_data, ) + raise ValueError(f"Error fetching data from BSE for {date}") + else: + del data[0] # delete title row + populate_bhav_copy_data_into_redis(data) + populate_bhav_copy_data_into_postgres(data, date=date) + + +def populate_bhav_copy_data_into_redis(data): + logger.info(f'Populating data into redis') + pipe = cache.pipeline() + cache.delete("stocks") + for stock in data: + # prevent creation of duplicate entries + pipe.rpush("stocks", stock[0]) + pipe.hset( + f"stock:{stock[0]}", + mapping={ + "sc_code": stock[0], + "sc_name": stock[1], + "sc_group": stock[2], + "sc_type": stock[3], + "open_price": float(stock[4]), + "high_price": float(stock[5]), + "low_price": float(stock[6]), + "close_price": float(stock[7]), + "last_price": float(stock[8]), + "prevclose_price": float(stock[9]), + "no_trades": int(stock[10]), + "no_of_shrs": int(stock[11]), + "net_turnov": float(stock[12]), + "tdcloindi": stock[13], + } + ) + pipe.execute() + + +@transaction.atomic +def populate_bhav_copy_data_into_postgres(data, date=None): + logger.info(f'Populating data into postgres for {date}') + if date is None: + date = datetime.datetime.now().date() + for stock in data: + BhavCopyEquity.objects.get_or_create( + date=date, + sc_code=int(stock[0]), + sc_name=stock[1], + sc_group=stock[2], + sc_type=stock[3], + open_price=float(stock[4]), + high_price=float(stock[5]), + low_price=float(stock[6]), + close_price=float(stock[7]), + last_price=float(stock[8]), + prevclose_price=float(stock[9]), + no_trades=int(stock[10]), + no_of_shrs=int(stock[11]), + net_turnov=float(stock[12]), + tdcloindi=stock[13], + ) diff --git a/backend/app/views.py b/backend/app/views.py index 12b6400..041ed6f 100644 --- a/backend/app/views.py +++ b/backend/app/views.py @@ -3,13 +3,15 @@ """Bullish page views.""" +# standard imports +import datetime +import logging # third-party imports -from rest_framework.response import Response from rest_framework import generics +from rest_framework.response import Response +from rest_framework.decorators import api_view from django_redis import get_redis_connection -from django.utils.decorators import method_decorator -from django.views.decorators.cache import cache_page # app imports from app.models import BhavCopyEquity @@ -18,16 +20,37 @@ from app.utils import populate_bhav_copy_data cache = get_redis_connection("default") +logger = logging.getLogger(__name__) # Create your views here. -class BhavCopyEquityView(generics.RetrieveAPIView): - queryset = BhavCopyEquity.objects.all() +@api_view(['GET']) +def bhavCopyEquityList(request): + ret_message = "" - def get(self, request, *args, **kwargs): - queryset = self.get_queryset() + # Verify Date + req_date = datetime.datetime.strptime(request.query_params.get('date'), '%Y-%m-%d') + # 18:00 IST == 12:30 UTC + today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0) + if req_date > today: + ret_message = "Time travel not yet invented! Returning latest available data." + req_date = today + if datetime.datetime.now() < today: + req_date = today - datetime.timedelta(days=1) + queryset = BhavCopyEquity.objects.all().filter(date=req_date) + serializer = BhavCopyEquitySerializer(queryset, many=True) + + # Fetch data if not present + if len(serializer.data) == 0: + logger.info(f'Data not available in DB') + populate_bhav_copy_data(date=req_date) + queryset = BhavCopyEquity.objects.all().filter(date=req_date) serializer = BhavCopyEquitySerializer(queryset, many=True) - return Response(serializer.data) + + return Response({ + "data": serializer.data, + "message": ret_message + }) class EmptyRespoinseView(generics.RetrieveAPIView): diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 61eeaa9..c95d179 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -163,6 +163,20 @@ SESSION_CACHE_ALIAS = "default" CACHE_TTL = 60 * 5 DJANGO_REDIS_CONNECTION_FACTORY = "app.redis.ConnectionFactory" +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'root': { + 'handlers': ['console'], + 'level': 'INFO', + }, +} + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ diff --git a/frontend/src/views/BhavCopy.vue b/frontend/src/views/BhavCopy.vue index 2c2edd7..abb105a 100644 --- a/frontend/src/views/BhavCopy.vue +++ b/frontend/src/views/BhavCopy.vue @@ -23,19 +23,36 @@ Bhav Copy - Equity - - Download - - mdi-cloud-download - - + + + + + + Download + + mdi-cloud-download + + + { console.log('BhavCopyEquity API has recieved data') - this.apiData = response.data - this.snackbarText = "Successfully loaded data" - this.snackbarColor = "green lighten-2" + this.apiData = response.data.data + this.snackbarText = response.data.message || "Successfully loaded data" + this.snackbarColor = `${response.data.message ? 'yellow' : 'green'} lighten-2` this.snackbar = true }) .catch(err => { console.log(err) - this.snackbarText = "Error fetching data" + this.snackbarText = "Server Error" this.snackbarColor = "red lighten-2" this.snackbar = true }) }, + formatDate (date) { + if (!date) return null + + const [year, month, day] = date.split('-') + return `${month}/${day}/${year}` + }, + parseDate (date) { + if (!date) return null + + const [month, day, year] = date.split('/') + return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}` + }, }, computed: { headers() { return this.headersData - } + }, + computedDateFormatted () { + return this.formatDate(this.date) + }, }, data() { return { sc_name: '', + date: new Date().toISOString().substr(0, 10), + dateFormatted: '', + datePickerMenu: false, snackbar: false, snackbarText: '', snackbarColor: '', apiData: [], - apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Custom Redis Cache" }, + apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" }, apiEndpoints: [ { endpoint: 'emptyresponse', text: "Empty Response Endpoint" }, { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" }, @@ -161,7 +214,12 @@ }, created() { this.populateApiData() - } + }, + watch: { + date () { + this.dateFormatted = this.formatDate(this.date) + }, + }, }