#!/usr/bin/env python # -*- coding: utf-8 -*- """Utils for bullish.""" # standard imports import csv import datetime from io import BytesIO, TextIOWrapper from zipfile import ZipFile # third-party imports import django_rq import requests from django_redis import get_redis_connection from django.db import transaction # app imports from app.models import BhavCopyEquity cache = get_redis_connection("default") def fetch_bhav_copy_equity_data(curr_date=None): """Fetch data from BSE India website.""" # hack: since bseindia website doesn't respond well to python requests user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0' headers = { 'User-Agent': user_agent } if curr_date is None: curr_date = datetime.datetime.now().strftime("%d%m%y") curr_date = '080221' # TODO: remove hardcode zipurl = f'https://www.bseindia.com/download/BhavCopy/Equity/EQ{curr_date}_CSV.ZIP' resp = requests.get(zipurl, headers=headers) if resp.ok: with ZipFile(BytesIO(resp.content)) as bhavcopy_zf: csv_file = bhavcopy_zf.namelist()[0] with bhavcopy_zf.open(csv_file, 'r') as infile: data = list(csv.reader(TextIOWrapper(infile, 'utf-8'))) return data return None @transaction.atomic def populate_bhav_copy_data(): """Populate DB with Bhav Copy data.""" try: pipe = cache.pipeline() data = fetch_bhav_copy_equity_data() del data[0] # delete title row cache.delete("stocks") for stock in data: # prevent creation of duplicate entries pipe.rpush("stocks", stock[0]) pipe.hset( f"stock:{stock[0]}", mapping={ "sc_code": stock[0], "sc_name": stock[1], "sc_group": stock[2], "sc_type": stock[3], "open_price": float(stock[4]), "high_price": float(stock[5]), "low_price": float(stock[6]), "close_price": float(stock[7]), "last_price": float(stock[8]), "prevclose_price": float(stock[9]), "no_trades": int(stock[10]), "no_of_shrs": int(stock[11]), "net_turnov": float(stock[12]), "tdcloindi": stock[13], } ) pipe.execute() BhavCopyEquity.objects.get_or_create( sc_code=int(stock[0]), sc_name=stock[1], sc_group=stock[2], sc_type=stock[3], open_price=float(stock[4]), high_price=float(stock[5]), low_price=float(stock[6]), close_price=float(stock[7]), last_price=float(stock[8]), prevclose_price=float(stock[9]), no_trades=int(stock[10]), no_of_shrs=int(stock[11]), net_turnov=float(stock[12]), tdcloindi=stock[13], ) except: # Potentially add code for alerting if needed # Repeat job after 10 mins if job fails scheduler = django_rq.get_scheduler('default') scheduler.schedule( scheduled_time=datetime.datetime.now()+datetime.timedelta(minutes=10), func=populate_bhav_copy_data, )