From 39edb20a4ddf05f74a31951d7e4e8e898cad8fba Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 16 Dec 2024 17:50:04 +0300 Subject: [PATCH 1/2] feat: add command to create selected indices --- backend/docker-compose.yaml | 3 +- .../management/commands/update_indices.py | 70 +++++++++++++++++++ backend/marketfeed/management/indices.json | 46 ++++++++++++ docker-compose.yaml | 1 + 4 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 backend/marketfeed/management/commands/update_indices.py create mode 100644 backend/marketfeed/management/indices.json diff --git a/backend/docker-compose.yaml b/backend/docker-compose.yaml index 4cefe877..e5ca53f0 100644 --- a/backend/docker-compose.yaml +++ b/backend/docker-compose.yaml @@ -33,7 +33,8 @@ services: python3 manage.py migrate --noinput && python3 manage.py collectstatic --noinput && python3 manage.py update_currencies && - python3 manage.py update_stocks && + python3 manage.py update_stocks && + python3 manage.py update_indices && gunicorn backend.wsgi:application --bind 0.0.0.0:8000' restart: always environment: diff --git a/backend/marketfeed/management/commands/update_indices.py b/backend/marketfeed/management/commands/update_indices.py new file mode 100644 index 00000000..c0ea0e9c --- /dev/null +++ b/backend/marketfeed/management/commands/update_indices.py @@ -0,0 +1,70 @@ +import requests +from bs4 import BeautifulSoup +from django.core.management.base import BaseCommand +import json +from django.conf import settings +import os +from marketfeed.serializers import * +from marketfeed.models import * + +class Command(BaseCommand): + help = 'Update or Insert the supported indices to db' + + def handle(self, *args, **kwargs): + url = "https://www.kap.org.tr/en/Endeksler" + + file_path = os.path.join(settings.BASE_DIR,'marketfeed', 'management', 'indices.json') + if not os.path.exists(file_path): + self.stdout.write(self.style.ERROR(f'Index data file not found: {file_path}')) + return + with open(file_path, 'r') as file: + indices = json.load(file) + response = requests.get(url) + response.raise_for_status() + + soup = BeautifulSoup(response.text, "html.parser") + + index_elements = soup.find_all("div", class_="vcell") + + for index_element in index_elements: + index_name = index_element.text.strip() + + if index_name not in indices: + continue + # Get the index name + next_div = index_element.find_next("div", class_="column-type7 wmargin") + stock_symbols = [] + if next_div: + stock_elements = next_div.find_all("div", class_="comp-cell _02 vtable") + for stock_element in stock_elements: + stock_name = stock_element.find("a", class_="vcell").text.strip() if stock_element.find("a", class_="vcell") else None + + if stock_name: + stock_symbols.append(stock_name) + + + + stocks_in_index = Stock.objects.filter(symbol__in=stock_symbols) + #stocks_in_index = list(stocks_in_index.values_list('id', flat=True)) + currency = Currency.objects.get(code='TRY') + symbol = indices[index_name] + + print(stocks_in_index) + print(currency) + print(symbol) + print(index_name) + + try: + index_obj, created = Index.objects.update_or_create( + name = index_name, + symbol = symbol, + currency = currency, + #stocks = stocks_in_index + ) + + index_obj.stocks.add(*stocks_in_index) + except Exception as e: + print(e) + + + \ No newline at end of file diff --git a/backend/marketfeed/management/indices.json b/backend/marketfeed/management/indices.json new file mode 100644 index 00000000..a74d9b63 --- /dev/null +++ b/backend/marketfeed/management/indices.json @@ -0,0 +1,46 @@ +{ + "BIST 100": "XU100", + "BIST 50": "XU050", + "BIST 30": "XU030", + "BIST LIQUID BANKS": "XLBNK", + "BIST LIQUID 10 EX BANKS": "X10XB", + "BIST STARS": "XYLDZ", + "BIST IPO": "XHARZ", + "BIST SME INDUSTRIAL": "XKOBI", + "BIST DIVIDEND 25": "XTM25", + "BIST CORPORATE GOVERNANCE": "XKURY", + "BIST SUSTAINABILITY 25": "XSD25", + "BIST INDUSTRIALS": "XUSIN", + "BIST FOOD, BEVERAGE": "XGIDA", + "BIST CHEM., PETROL, PLASTIC": "XKMYA", + "BIST MINING": "XMADN", + "BIST BASIC METAL": "XMANA", + "BIST METAL PRODUCTS, MACH.": "XMESY", + "BIST WOOD, PAPER, PRINTING": "XKAGT", + "BIST NON-METAL MIN. PRODUCT": "XTAST", + "BIST TEXTILE, LEATHER": "XTEKS", + "BIST SERVICES": "XUHIZ", + "BIST ELECTRICITY": "XELKT", + "BIST TELECOMMUNICATION": "XILTM", + "BIST CONSTRUCTION": "XINSA", + "BIST SPORTS": "XSPOR", + "BIST W. AND RETAIL TRADE": "XTCRT", + "BIST TOURISM": "XTRZM", + "BIST TRANSPORTATION": "XULAS", + "BIST FINANCIALS": "XUMAL", + "BIST BANKS": "XBANK", + "BIST INSURANCE": "XSGRT", + "BIST LEASING FACTORING": "XFINK", + "BIST HOLD. AND INVESTMENT": "XHOLD", + "BIST REAL EST. INV. TRUSTS": "XGMYO", + "BIST BROKERAGE HOUSES": "XAKUR", + "BIST INVESTMENT TRUSTS": "XYORT", + "BIST TECHNOLOGY": "XUTEK", + "BIST INF. TECHNOLOGY": "XBLSM", + "BIST PARTICIPATION ALL SHARES": "XKTUM", + "BIST PARTICIPATION 100": "XK100", + "BIST PARTICIPATION DIVIDEND": "XKTMT", + "BIST SUSTAINABILITY PARTICIPATION": "XSRDK", + "BIST BUYBACK": "XUGRA" + } + \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 1b8c1c77..28890208 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -31,6 +31,7 @@ services: python3 manage.py collectstatic --noinput && python3 manage.py update_currencies && python3 manage.py update_stocks && + python3 manage.py update_indices && gunicorn backend.wsgi:application --bind 0.0.0.0:8000' restart: always environment: From de6f5610cfd47d9137e31d9514636cc838fa3789 Mon Sep 17 00:00:00 2001 From: cemgungor1 Date: Mon, 16 Dec 2024 17:51:33 +0300 Subject: [PATCH 2/2] fix: return error on index get and cleaner code --- .../marketfeed/management/commands/update_indices.py | 11 ++--------- backend/marketfeed/views.py | 3 ++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/backend/marketfeed/management/commands/update_indices.py b/backend/marketfeed/management/commands/update_indices.py index c0ea0e9c..7417968e 100644 --- a/backend/marketfeed/management/commands/update_indices.py +++ b/backend/marketfeed/management/commands/update_indices.py @@ -40,20 +40,13 @@ def handle(self, *args, **kwargs): stock_name = stock_element.find("a", class_="vcell").text.strip() if stock_element.find("a", class_="vcell") else None if stock_name: - stock_symbols.append(stock_name) - - + stock_symbols.append(stock_name) stocks_in_index = Stock.objects.filter(symbol__in=stock_symbols) - #stocks_in_index = list(stocks_in_index.values_list('id', flat=True)) currency = Currency.objects.get(code='TRY') symbol = indices[index_name] - print(stocks_in_index) - print(currency) - print(symbol) - print(index_name) - + try: index_obj, created = Index.objects.update_or_create( name = index_name, diff --git a/backend/marketfeed/views.py b/backend/marketfeed/views.py index 0f9a2cc8..6fb1e735 100644 --- a/backend/marketfeed/views.py +++ b/backend/marketfeed/views.py @@ -628,8 +628,9 @@ def list(self, request): ) for index in serializerData ] + if not symbols: + return Response([], status=status.HTTP_200_OK) data = yf.download(tickers=symbols, period="1d", interval="1d") - prices = { symbol.split(".")[0]: float(data["Close"][symbol]) for symbol in symbols }