feat: get rq working and schedule job at 6

Signed-off-by: Ameya Shenoy <shenoy.ameya@gmail.com>

backend/Dockerfile

@@ -22,5 +22,5 @@ RUN set -ex \
 
 COPY . /code
 
-ENTRYPOINT sh /code/entrypoint.sh
+CMD sh /code/entrypoint.sh
 

backend/app/__init__.py

@@ -0,0 +1,2 @@
+default_app_config = 'app.apps.AppConfig'
+

backend/app/apps.py

@@ -1,5 +1,36 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Initializer and scheduling done here."""
+
+# standard imports
+import datetime
+import logging
+
+# third-party imports
+import django_rq
+
 from django.apps import AppConfig
+from django_rq.management.commands import rqscheduler
 
 
 class AppConfig(AppConfig):
     name = 'app'
+
+    def ready(self):
+        from app.utils import populate_bhav_copy_data, get_next_update_datetime
+
+        # This is necessary to prevent dupes
+        scheduler = django_rq.get_scheduler('default')
+        # Delete any existing jobs in the scheduler when the app starts up
+        for job in scheduler.get_jobs():
+            job.delete()
+        # Schedule jobs here as required
+        next_date_time = get_next_update_datetime()
+        scheduler.schedule(
+            scheduled_time=next_date_time,  # UTC
+            func=populate_bhav_copy_data,
+            interval=86400,
+        )
+
+

backend/app/migrations/0002_auto_20210209_0920.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.1.6 on 2021-02-09 09:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bhavcopyequity',
+            name='net_turnov',
+            field=models.DecimalField(decimal_places=2, max_digits=12),
+        ),
+        migrations.AlterField(
+            model_name='bhavcopyequity',
+            name='tdcloindi',
+            field=models.TextField(blank=True),
+        ),
+    ]

backend/app/models.py

@@ -19,7 +19,7 @@ class BhavCopyEquity(models.Model):
     prevclose_price = models.DecimalField(max_digits=12, decimal_places=2)
     no_trades = models.PositiveIntegerField()
     no_of_shrs = models.PositiveIntegerField()
-    net_turnov = models.PositiveIntegerField()
+    net_turnov = models.DecimalField(max_digits=12, decimal_places=2)
     tdcloindi = models.TextField(blank=True)
 
     def __str__(self):

backend/app/utils.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Utils for bullish."""
+
+# standard imports
+import csv
+import datetime
+
+from io import BytesIO, TextIOWrapper
+from zipfile import ZipFile
+
+# third-party imports
+import django_rq
+import requests
+
+from django.db import transaction
+from django_rq import job
+
+# app imports
+from app.models import BhavCopyEquity
+
+
+def fetch_bhav_copy_equity_data(curr_date=None):
+    """Fetch data from BSE India website."""
+    # hack: since bseindia website doesn't respond well to python requests
+    user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0'
+    headers = {
+        'User-Agent': user_agent
+    }
+    if curr_date is None:
+        curr_date = datetime.datetime.now().strftime("%d%m%y")
+    curr_date = '080221'  # TODO: remove hardcode
+    zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP'
+    resp = requests.get(zipurl, headers=headers)
+
+    if resp.ok:
+        with ZipFile(BytesIO(resp.content)) as bhavcopy_zf:
+            csv_file = bhavcopy_zf.namelist()[0]
+            with bhavcopy_zf.open(csv_file, 'r') as infile:
+                data = list(csv.reader(TextIOWrapper(infile, 'utf-8')))  # TODO: potentially remove list
+        return data
+    # schedule again in 5 minutes
+    return None
+
+
+@transaction.atomic
+def populate_bhav_copy_data():
+    """Populate DB with Bhav Copy data."""
+    data = fetch_bhav_copy_equity_data()
+    del data[0]
+    for stock in data:
+        # prevent creation of duplicate entries
+        BhavCopyEquity.objects.get_or_create(
+            sc_code=int(stock[0]),
+            sc_name=stock[1],
+            sc_group=stock[2],
+            sc_type=stock[3],
+            open_price=float(stock[4]),
+            high_price=float(stock[5]),
+            low_price=float(stock[6]),
+            close_price=float(stock[7]),
+            last_price=float(stock[8]),
+            prevclose_price=float(stock[9]),
+            no_trades=int(stock[10]),
+            no_of_shrs=int(stock[11]),
+            net_turnov=float(stock[12]),
+            tdcloindi=stock[13],
+        )
+
+
+def get_next_update_datetime(date=None):
+    schedule_hour = 12
+    schedule_minute = 30
+    next_date_time = now = datetime.datetime.now()
+    if date:
+        next_date_time = now = date
+    if now.hour >= schedule_hour and now.minute >= schedule_minute:
+        next_date_time = now + datetime.timedelta(days=1)
+    next_date_time = next_date_time.replace(
+        hour=schedule_hour,
+        minute=schedule_minute,
+        second=0,
+        microsecond=0
+    )
+    return next_date_time
+
+

backend/backend/settings.py

@@ -39,6 +39,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'corsheaders',
+    'django_rq',
     'app',
 ]
 
@@ -94,7 +95,15 @@ DATABASES = {
     }
 }
 
-
+CACHES = {
+    'default': {
+        'BACKEND': 'redis_cache.cache.RedisCache',
+        'LOCATION': 'redis:6379/1',
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+        },
+    },
+}
 
 # Password validation
 # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
@@ -134,6 +143,16 @@ CORS_ALLOWED_ORIGINS = [
     "http://127.0.0.1:8080",
 ]
 
+RQ_QUEUES = {
+    'default': {
+        'HOST': 'redis',
+        'PORT': 6379,
+        'DB': 0,
+        'DEFAULT_TIMEOUT': 360,
+    },
+}
+
+
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 

backend/backend/urls.py

@@ -18,5 +18,6 @@ from django.urls import path, include
 
 urlpatterns = [
     path('admin/', admin.site.urls),
-    path('', include('app.urls'))
+    path('django-rq/', include('django_rq.urls')),
+    path('', include('app.urls')),
 ]

docker-compose.yml

@@ -11,6 +11,8 @@ services:
       - ./backend:/code
     depends_on:
       - db
+      - rqworker
+      - rqscheduler
       - redis
     ports:
       - 8000:8000
@@ -29,6 +31,26 @@ services:
     # ports:
     #   - 6379:6379
 
+  rqworker:
+    image: codingcoffee/bullish-backend
+    build:
+      context: ./backend
+    volumes:
+      - ./backend:/code
+    command: python manage.py rqworker default
+    depends_on:
+      - redis
+
+  rqscheduler:
+    image: codingcoffee/bullish-backend
+    build:
+      context: ./backend
+    volumes:
+      - ./backend:/code
+    command: python manage.py rqscheduler
+    depends_on:
+      - redis
+
   db:
     image: postgres:13.1-alpine
     restart: unless-stopped