codingcoffee/bullish
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

feat: implement date picker

Browse source 
- re write postgres endpoint
- enable logging in django
- ui for date picker

Signed-off-by: Ameya Shenoy <shenoy.ameya@gmail.com>
This commit is contained in:
Ameya Shenoy 2021-02-13 02:08:40 +05:30
parent 563a23a1d9
commit 2b9bfd1ddb
Signed by: codingcoffee
GPG key ID: F7D58AAC5DACF8D3
9 changed files with 256 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
README.md
View file

@ -59,6 +59,28 @@ docker logs -f container-name
Here `container-name` can be either of `backend`, `frontend`, `redis`,
`postgresql`, `rqworker` or `rqscheduler`.
### Debugging
- To goto Django's shell to debug you may use
```sh
docker-compose exec backend python manage.py shell
```
- To create migrations
```sh
docker-compose exec backend python manage.py makemigrations
```
- To run migrate
```sh
docker-compose exec backend python manage.py migrate
```
Similarly all django commands can be run this way.
## Deploy

18
backend/app/migrations/0003_auto_20210212_1348.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.1.6 on 2021-02-12 13:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app', '0002_auto_20210209_0920'),
]
operations = [
migrations.AlterField(
model_name='bhavcopyequity',
name='date',
field=models.DateField(editable=False),
),
]

18
backend/app/migrations/0004_auto_20210212_1846.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.1.6 on 2021-02-12 18:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('app', '0003_auto_20210212_1348'),
]
operations = [
migrations.AlterField(
model_name='bhavcopyequity',
name='date',
field=models.DateField(),
),
]

2
backend/app/models.py
View file

@ -4,7 +4,7 @@ from django.db import models
class BhavCopyEquity(models.Model):
id = models.AutoField(primary_key=True)
date = models.DateField(auto_now_add=True, editable=False)
date = models.DateField()
# Ref: https://www.bseindia.com/markets/debt/BhavCopyhelp.aspx
sc_code = models.PositiveIntegerField()

6
backend/app/urls.py
View file

@ -5,15 +5,15 @@
from django.urls import path
from app import views
from app.views import (
BhavCopyEquityView,
EmptyRespoinseView,
BhavCopyEquityCustomRedisView,
EmptyRespoinseView
)
urlpatterns = [
path('emptyresponse/', EmptyRespoinseView.as_view(), name='emptyresponse_view'),
path('bhavcopyequity/', BhavCopyEquityView.as_view(), name='bhavcopyequity_view'),
path('bhavcopyequity/', views.bhavCopyEquityList, name='bhavcopyequity_view'),
path(
'bhavcopyequitycustomredis/',
BhavCopyEquityCustomRedisView.as_view(),

120
backend/app/utils.py
View file

@ -6,6 +6,7 @@
# standard imports
import csv
import datetime
import logging
from io import BytesIO, TextIOWrapper
from zipfile import ZipFile
@ -22,18 +23,20 @@ from app.models import BhavCopyEquity
cache = get_redis_connection("default")
logger = logging.getLogger(__name__)
def fetch_bhav_copy_equity_data(curr_date=None):
def fetch_bhav_copy_equity_data(date=None):
"""Fetch data from BSE India website."""
# hack: since bseindia website doesn't respond well to python requests
user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0'
headers = {
'User-Agent': user_agent
}
if curr_date is None:
curr_date = datetime.datetime.now().strftime("%d%m%y")
zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP'
if date is None:
date = datetime.datetime.now()
zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{date.strftime("%d%m%y")}_CSV.ZIP'
logger.info(f'Fetching data from {zipurl}')
resp = requests.get(zipurl, headers=headers)
if resp.ok:
@ -42,57 +45,13 @@ def fetch_bhav_copy_equity_data(curr_date=None):
with bhavcopy_zf.open(csv_file, 'r') as infile:
data = list(csv.reader(TextIOWrapper(infile, 'utf-8')))
return data
return None
raise ValueError('Fetching data from BSE unsuccessful')
@transaction.atomic
def populate_bhav_copy_data():
def populate_bhav_copy_data(date=None):
"""Populate DB with Bhav Copy data."""
try:
pipe = cache.pipeline()
data = fetch_bhav_copy_equity_data()
del data[0] # delete title row
cache.delete("stocks")
for stock in data:
# prevent creation of duplicate entries
pipe.rpush("stocks", stock[0])
pipe.hset(
f"stock:{stock[0]}",
mapping={
"sc_code": stock[0],
"sc_name": stock[1],
"sc_group": stock[2],
"sc_type": stock[3],
"open_price": float(stock[4]),
"high_price": float(stock[5]),
"low_price": float(stock[6]),
"close_price": float(stock[7]),
"last_price": float(stock[8]),
"prevclose_price": float(stock[9]),
"no_trades": int(stock[10]),
"no_of_shrs": int(stock[11]),
"net_turnov": float(stock[12]),
"tdcloindi": stock[13],
}
)
pipe.execute()
BhavCopyEquity.objects.get_or_create(
sc_code=int(stock[0]),
sc_name=stock[1],
sc_group=stock[2],
sc_type=stock[3],
open_price=float(stock[4]),
high_price=float(stock[5]),
low_price=float(stock[6]),
close_price=float(stock[7]),
last_price=float(stock[8]),
prevclose_price=float(stock[9]),
no_trades=int(stock[10]),
no_of_shrs=int(stock[11]),
net_turnov=float(stock[12]),
tdcloindi=stock[13],
)
data = fetch_bhav_copy_equity_data(date=date)
except:
# Potentially add code for alerting if needed
# Repeat job after 10 mins if job fails
@ -101,4 +60,63 @@ def populate_bhav_copy_data():
scheduled_time=datetime.datetime.now()+datetime.timedelta(minutes=10),
func=populate_bhav_copy_data,
)
raise ValueError(f"Error fetching data from BSE for {date}")
else:
del data[0] # delete title row
populate_bhav_copy_data_into_redis(data)
populate_bhav_copy_data_into_postgres(data, date=date)
def populate_bhav_copy_data_into_redis(data):
logger.info(f'Populating data into redis')
pipe = cache.pipeline()
cache.delete("stocks")
for stock in data:
# prevent creation of duplicate entries
pipe.rpush("stocks", stock[0])
pipe.hset(
f"stock:{stock[0]}",
mapping={
"sc_code": stock[0],
"sc_name": stock[1],
"sc_group": stock[2],
"sc_type": stock[3],
"open_price": float(stock[4]),
"high_price": float(stock[5]),
"low_price": float(stock[6]),
"close_price": float(stock[7]),
"last_price": float(stock[8]),
"prevclose_price": float(stock[9]),
"no_trades": int(stock[10]),
"no_of_shrs": int(stock[11]),
"net_turnov": float(stock[12]),
"tdcloindi": stock[13],
}
)
pipe.execute()
@transaction.atomic
def populate_bhav_copy_data_into_postgres(data, date=None):
logger.info(f'Populating data into postgres for {date}')
if date is None:
date = datetime.datetime.now().date()
for stock in data:
BhavCopyEquity.objects.get_or_create(
date=date,
sc_code=int(stock[0]),
sc_name=stock[1],
sc_group=stock[2],
sc_type=stock[3],
open_price=float(stock[4]),
high_price=float(stock[5]),
low_price=float(stock[6]),
close_price=float(stock[7]),
last_price=float(stock[8]),
prevclose_price=float(stock[9]),
no_trades=int(stock[10]),
no_of_shrs=int(stock[11]),
net_turnov=float(stock[12]),
tdcloindi=stock[13],
)

39
backend/app/views.py
View file

@ -3,13 +3,15 @@
"""Bullish page views."""
# standard imports
import datetime
import logging
# third-party imports
from rest_framework.response import Response
from rest_framework import generics
from rest_framework.response import Response
from rest_framework.decorators import api_view
from django_redis import get_redis_connection
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
# app imports
from app.models import BhavCopyEquity
@ -18,16 +20,37 @@ from app.utils import populate_bhav_copy_data
cache = get_redis_connection("default")
logger = logging.getLogger(__name__)
# Create your views here.
class BhavCopyEquityView(generics.RetrieveAPIView):
queryset = BhavCopyEquity.objects.all()
@api_view(['GET'])
def bhavCopyEquityList(request):
ret_message = ""
def get(self, request, *args, **kwargs):
queryset = self.get_queryset()
# Verify Date
req_date = datetime.datetime.strptime(request.query_params.get('date'), '%Y-%m-%d')
# 18:00 IST == 12:30 UTC
today = datetime.datetime.now().replace(hour=12, minute=30, second=0, microsecond=0)
if req_date > today:
ret_message = "Time travel not yet invented! Returning latest available data."
req_date = today
if datetime.datetime.now() < today:
req_date = today - datetime.timedelta(days=1)
queryset = BhavCopyEquity.objects.all().filter(date=req_date)
serializer = BhavCopyEquitySerializer(queryset, many=True)
# Fetch data if not present
if len(serializer.data) == 0:
logger.info(f'Data not available in DB')
populate_bhav_copy_data(date=req_date)
queryset = BhavCopyEquity.objects.all().filter(date=req_date)
serializer = BhavCopyEquitySerializer(queryset, many=True)
return Response(serializer.data)
return Response({
"data": serializer.data,
"message": ret_message
})
class EmptyRespoinseView(generics.RetrieveAPIView):

14
backend/backend/settings.py
View file

@ -163,6 +163,20 @@ SESSION_CACHE_ALIAS = "default"
CACHE_TTL = 60 * 5
DJANGO_REDIS_CONNECTION_FACTORY = "app.redis.ConnectionFactory"
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
}
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/

102
frontend/src/views/BhavCopy.vue
View file

@ -23,19 +23,36 @@
<v-card-title>
Bhav Copy - Equity
<v-spacer></v-spacer>
<v-btn
class="ma-2"
color="blue lighten-2"
@click="downloadcsv"
>
Download
<v-icon
light
right
>
mdi-cloud-download
</v-icon>
</v-btn>
<v-menu
ref="datePickerMenu"
v-model="datePickerMenu"
:close-on-content-click="false"
transition="scale-transition"
offset-y
max-width="290px"
min-width="auto"
>
<template v-slot:activator="{ on, attrs }">
<v-text-field
v-model="dateFormatted"
label="Date"
hint="MM/DD/YYYY format"
persistent-hint
prepend-icon="mdi-calendar"
v-bind="attrs"
@blur="date = parseDate(dateFormatted)"
v-on="on"
></v-text-field>
</template>
<v-date-picker
v-model="date"
no-title
v-on:change="populateApiData"
@input="datePickerMenu = false"
min="2007-01-02"
v-bind:max="(new Date()).toISOString().slice(0, 10)"
></v-date-picker>
</v-menu>
<v-spacer></v-spacer>
<v-select
v-model="apiEndpointSelected.endpoint"
@ -45,6 +62,20 @@
item-value="endpoint"
></v-select>
<v-spacer></v-spacer>
<v-btn
class="ma-2"
color="blue lighten-2"
@click="downloadcsv"
>
Download
<v-icon
light
right
>
mdi-cloud-download
</v-icon>
</v-btn>
<v-spacer></v-spacer>
<v-text-field
ref="searchField"
v-model="sc_name"
@ -100,36 +131,58 @@
},
populateApiData() {
var endpoint = this.apiEndpointSelected.endpoint;
console.log(`Fetching data from ${endpoint} API`)
getAPI.get(`/${endpoint}/`,)
console.log(`Fetching data from ${endpoint} API for ${this.date} date`)
getAPI.get(`/${endpoint}/`, {
params: {
date: this.date
}
})
.then(response => {
console.log('BhavCopyEquity API has recieved data')
this.apiData = response.data
this.snackbarText = "Successfully loaded data"
this.snackbarColor = "green lighten-2"
this.apiData = response.data.data
this.snackbarText = response.data.message || "Successfully loaded data"
this.snackbarColor = `${response.data.message ? 'yellow' : 'green'} lighten-2`
this.snackbar = true
})
.catch(err => {
console.log(err)
this.snackbarText = "Error fetching data"
this.snackbarText = "Server Error"
this.snackbarColor = "red lighten-2"
this.snackbar = true
})
},
formatDate (date) {
if (!date) return null
const [year, month, day] = date.split('-')
return `${month}/${day}/${year}`
},
parseDate (date) {
if (!date) return null
const [month, day, year] = date.split('/')
return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}`
},
},
computed: {
headers() {
return this.headersData
}
},
computedDateFormatted () {
return this.formatDate(this.date)
},
},
data() {
return {
sc_name: '',
date: new Date().toISOString().substr(0, 10),
dateFormatted: '',
datePickerMenu: false,
snackbar: false,
snackbarText: '',
snackbarColor: '',
apiData: [],
apiEndpointSelected: { endpoint: 'bhavcopyequitycustomredis', text: "Custom Redis Cache" },
apiEndpointSelected: { endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },
apiEndpoints: [
{ endpoint: 'emptyresponse', text: "Empty Response Endpoint" },
{ endpoint: 'bhavcopyequity', text: "Postgres Endpoint" },
@ -161,7 +214,12 @@
},
created() {
this.populateApiData()
}
},
watch: {
date () {
this.dateFormatted = this.formatDate(this.date)
},
},
}
</script>