diff --git a/backend/Dockerfile b/backend/Dockerfile index 17fd30d..4ede12b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -22,5 +22,5 @@ RUN set -ex \ COPY . /code -ENTRYPOINT sh /code/entrypoint.sh +CMD sh /code/entrypoint.sh diff --git a/backend/app/__init__.py b/backend/app/__init__.py index e69de29..082514a 100644 --- a/backend/app/__init__.py +++ b/backend/app/__init__.py @@ -0,0 +1,2 @@ +default_app_config = 'app.apps.AppConfig' + diff --git a/backend/app/apps.py b/backend/app/apps.py index 80b2c8d..9cace7b 100644 --- a/backend/app/apps.py +++ b/backend/app/apps.py @@ -1,5 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Initializer and scheduling done here.""" + +# standard imports +import datetime +import logging + +# third-party imports +import django_rq + from django.apps import AppConfig +from django_rq.management.commands import rqscheduler class AppConfig(AppConfig): name = 'app' + + def ready(self): + from app.utils import populate_bhav_copy_data, get_next_update_datetime + + # This is necessary to prevent dupes + scheduler = django_rq.get_scheduler('default') + # Delete any existing jobs in the scheduler when the app starts up + for job in scheduler.get_jobs(): + job.delete() + # Schedule jobs here as required + next_date_time = get_next_update_datetime() + scheduler.schedule( + scheduled_time=next_date_time, # UTC + func=populate_bhav_copy_data, + interval=86400, + ) + + diff --git a/backend/app/migrations/0002_auto_20210209_0920.py b/backend/app/migrations/0002_auto_20210209_0920.py new file mode 100644 index 0000000..492ffc4 --- /dev/null +++ b/backend/app/migrations/0002_auto_20210209_0920.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.6 on 2021-02-09 09:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='bhavcopyequity', + name='net_turnov', + field=models.DecimalField(decimal_places=2, max_digits=12), + ), + migrations.AlterField( + model_name='bhavcopyequity', + name='tdcloindi', + field=models.TextField(blank=True), + ), + ] diff --git a/backend/app/models.py b/backend/app/models.py index 3e11bbb..53fc5f8 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -19,7 +19,7 @@ class BhavCopyEquity(models.Model): prevclose_price = models.DecimalField(max_digits=12, decimal_places=2) no_trades = models.PositiveIntegerField() no_of_shrs = models.PositiveIntegerField() - net_turnov = models.PositiveIntegerField() + net_turnov = models.DecimalField(max_digits=12, decimal_places=2) tdcloindi = models.TextField(blank=True) def __str__(self): diff --git a/backend/app/utils.py b/backend/app/utils.py new file mode 100644 index 0000000..cfbb9b1 --- /dev/null +++ b/backend/app/utils.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Utils for bullish.""" + +# standard imports +import csv +import datetime + +from io import BytesIO, TextIOWrapper +from zipfile import ZipFile + +# third-party imports +import django_rq +import requests + +from django.db import transaction +from django_rq import job + +# app imports +from app.models import BhavCopyEquity + + +def fetch_bhav_copy_equity_data(curr_date=None): + """Fetch data from BSE India website.""" + # hack: since bseindia website doesn't respond well to python requests + user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0' + headers = { + 'User-Agent': user_agent + } + if curr_date is None: + curr_date = datetime.datetime.now().strftime("%d%m%y") + curr_date = '080221' # TODO: remove hardcode + zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP' + resp = requests.get(zipurl, headers=headers) + + if resp.ok: + with ZipFile(BytesIO(resp.content)) as bhavcopy_zf: + csv_file = bhavcopy_zf.namelist()[0] + with bhavcopy_zf.open(csv_file, 'r') as infile: + data = list(csv.reader(TextIOWrapper(infile, 'utf-8'))) # TODO: potentially remove list + return data + # schedule again in 5 minutes + return None + + +@transaction.atomic +def populate_bhav_copy_data(): + """Populate DB with Bhav Copy data.""" + data = fetch_bhav_copy_equity_data() + del data[0] + for stock in data: + # prevent creation of duplicate entries + BhavCopyEquity.objects.get_or_create( + sc_code=int(stock[0]), + sc_name=stock[1], + sc_group=stock[2], + sc_type=stock[3], + open_price=float(stock[4]), + high_price=float(stock[5]), + low_price=float(stock[6]), + close_price=float(stock[7]), + last_price=float(stock[8]), + prevclose_price=float(stock[9]), + no_trades=int(stock[10]), + no_of_shrs=int(stock[11]), + net_turnov=float(stock[12]), + tdcloindi=stock[13], + ) + + +def get_next_update_datetime(date=None): + schedule_hour = 12 + schedule_minute = 30 + next_date_time = now = datetime.datetime.now() + if date: + next_date_time = now = date + if now.hour >= schedule_hour and now.minute >= schedule_minute: + next_date_time = now + datetime.timedelta(days=1) + next_date_time = next_date_time.replace( + hour=schedule_hour, + minute=schedule_minute, + second=0, + microsecond=0 + ) + return next_date_time + + diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 26a0af1..5f86446 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -39,6 +39,7 @@ INSTALLED_APPS = [ 'django.contrib.messages', 'django.contrib.staticfiles', 'corsheaders', + 'django_rq', 'app', ] @@ -94,7 +95,15 @@ DATABASES = { } } - +CACHES = { + 'default': { + 'BACKEND': 'redis_cache.cache.RedisCache', + 'LOCATION': 'redis:6379/1', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + }, + }, +} # Password validation # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators @@ -134,6 +143,16 @@ CORS_ALLOWED_ORIGINS = [ "http://127.0.0.1:8080", ] +RQ_QUEUES = { + 'default': { + 'HOST': 'redis', + 'PORT': 6379, + 'DB': 0, + 'DEFAULT_TIMEOUT': 360, + }, +} + + # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ diff --git a/backend/backend/urls.py b/backend/backend/urls.py index beb454d..4eafe93 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -18,5 +18,6 @@ from django.urls import path, include urlpatterns = [ path('admin/', admin.site.urls), - path('', include('app.urls')) + path('django-rq/', include('django_rq.urls')), + path('', include('app.urls')), ] diff --git a/docker-compose.yml b/docker-compose.yml index 2a42c54..b2c8f5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: - ./backend:/code depends_on: - db + - rqworker + - rqscheduler - redis ports: - 8000:8000 @@ -29,6 +31,26 @@ services: # ports: # - 6379:6379 + rqworker: + image: codingcoffee/bullish-backend + build: + context: ./backend + volumes: + - ./backend:/code + command: python manage.py rqworker default + depends_on: + - redis + + rqscheduler: + image: codingcoffee/bullish-backend + build: + context: ./backend + volumes: + - ./backend:/code + command: python manage.py rqscheduler + depends_on: + - redis + db: image: postgres:13.1-alpine restart: unless-stopped