Sp airflow setup

.gitignore

@@ -5,4 +5,6 @@ __pycache__
 examples
 exploration
 .env
-.venv
+.venv
+backend/airflow/database
+.ipynb_checkpoints/

backend/.env.template

@@ -1,3 +1,8 @@
 JUICE_DB_NAME=""
 JUICE_DB_USER=""
-JUICE_DB_PASSWORD=""
+JUICE_DB_PASSWORD=""
+AIRFLOW_EMAIL=""
+AIRFLOW_FIRSTNAME=""
+AIRFLOW_LASTNAME=""
+AIRFLOW_PASSWORD=""
+AIRFLOW_USERNAME=""

backend/airflow/Dockerfile

@@ -0,0 +1,5 @@
+FROM apache/airflow:2.1.2-python3.8
+
+COPY --chown=airflow ./startup/* /usr/bin/ 
+
+ENTRYPOINT [ "/bin/sh", "-c" ]

backend/airflow/dags/get_games.py

@@ -0,0 +1,11 @@
+import pandas as pd
+from datetime import datetime
+
+def get_team_id_mapping():
+    teams = pd.read_sql('airflow/data/teams.csv')
+    teams = teams[teams.active][['team_id', 'city', 'mascot']]
+    teams_mapping = {
+        f'{city} {mascot}': team_id for (_, (team_id, city, mascot)) in teams.iterrows()
+    }
+    return teams_mapping
+

backend/airflow/dags/upload_teams.py

@@ -0,0 +1,36 @@
+from airflow.decorators import dag, task
+from airflow.utils.dates import days_ago
+import pandas as pd
+import requests
+import os
+
+default_args = {
+    'owner': 'sampocs'
+}
+
+API_ENDPOINT = os.environ['JUICE_API_ENDPOINT']
+
+@dag(default_args=default_args, schedule_interval=None, start_date=days_ago(1))
+def upload_teams():
+
+    @task()
+    def upload():
+        print('Reading data into pandas...')
+        df = pd.read_csv('/opt/airflow/data/teams.csv')
+        rows = df.to_dict('records')
+
+        print('Writing to database...')
+        res = requests.post(f'{API_ENDPOINT}/teams', json=rows)
+
+        assert res.status_code == 200, f'API Error {res.status_code}'
+
+        print('Response:')
+        print(res.json())
+
+        print('Done.')
+
+        return 
+
+    upload()
+
+upload_teams = upload_teams()

backend/airflow/data/teams.csv

@@ -0,0 +1,37 @@
+team_id,org_id,city,mascot,start_year,active,pfr_name
+ARI,ARI,Arizona,Cardinals,1994,TRUE,crd
+ATL,ATL,Atlanta,Falcons,1966,TRUE,atl
+BAL,BAL,Baltimore,Ravens,1996,TRUE,rav
+BUF,BUF,Buffalo,Bills,1960,TRUE,buf
+CAR,CAR,Carolina,Panthers,1995,TRUE,car
+CHI,CHI,Chicago,Bears,1922,TRUE,chi
+CIN,CIN,Cincinnati,Bengals,1968,TRUE,cin
+CLE,CLE,Cleveland,Browns,1946,TRUE,cle
+DAL,DAL,Dallas,Cowboys,1960,TRUE,dal
+DEN,DEN,Denver,Broncos,1960,TRUE,den
+DET,DET,Detroit,Lions,1934,TRUE,det
+GB,GB,Green Bay,Packers,1919,TRUE,gnb
+HOU,HOU,Houston,Texans,2002,TRUE,htx
+IND,IND,Indianapolis,Colts,1984,TRUE,clt
+JAX,JAX,Jacksonville,Jaguars,1995,TRUE,jax
+KC,KC,Kansas City,Chiefs,1963,TRUE,kan
+LV,LV,Las Vegas,Raiders,2020,TRUE,rai
+LAC,LAC,Los Angeles,Chargers,2017,TRUE,sdg
+LAR,LAR,Los Angeles,Rams,2016,TRUE,ram
+MIA,MIA,Miami,Dolphins,1966,TRUE,mia
+MIN,MIN,Minnesota,Vikings,1961,TRUE,min
+NE,NE,New England,Patriots,1971,TRUE,nwe
+NO,NO,New Orleans,Saints,1967,TRUE,nor
+NYG,NYG,New York,Giants,1925,TRUE,nyg
+NYJ,NYJ,New York,Jets,1963,TRUE,nyj
+PHI,PHI,Philadelphia,Eagles,1944,TRUE,phi
+PIT,PIT,Pittsburgh,Steelers,1945,TRUE,pit
+SF,SF,San Francisco,49ers,1946,TRUE,sfo
+SEA,SEA,Seattle,Seahawks,1976,TRUE,sea
+TB,TB,Tampa Bay,Buccaneers,1976,TRUE,tam
+TEN,TEN,Tennessee,Titans,1999,TRUE,oti
+WSH,WSH,Washington,Football Team,2020,TRUE,was
+OAK,LV,Oakland,Raiders,1995,FALSE,rai
+STL,LAR,St Louis,Rams,1995,FALSE,ram
+SD,LAC,San Diego,Chargers,1961,FALSE,sdg
+WSHR,WSH,Washington,Redskins,1937,FALSE,was

backend/airflow/startup/start-scheduler.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [[ -f "/opt/airflow/database/airflow.db" ]];
+then
+    echo "Database already initialized."
+else
+    echo "Initializing DB..."
+    airflow db init 
+fi
+
+# Create a main user if it hasn't been done already
+if [[ "$(airflow users list | grep ${AIRFLOW_USERNAME} | wc -l)" == "0" ]]; then
+    echo "Creating user..."
+    airflow users create \
+        -u ${AIRFLOW_USERNAME} \
+        -e ${AIRFLOW_EMAIL} \
+        -f ${AIRFLOW_FIRSTNAME} \
+        -l ${AIRFLOW_LASTNAME} \
+        -p ${AIRFLOW_PASSWORD} \
+        -r Admin
+else
+    echo "User: ${AIRFLOW_USERNAME} already exists."
+fi
+
+if [[ "$(airflow connections get juice-postgres)" == "" ]]; then 
+    echo "Creating juice-db connection..."
+    airflow connections add "juice-postgres" \
+        --conn-uri "postgresql://${JUICE_DB_USER}:${JUICE_DB_PASSWORD}@db:5432/${JUICE_DB_NAME}"
+else
+    echo "juice-postgres connection already exists."
+fi
+
+echo "Starting scheduler..."
+airflow scheduler 

backend/airflow/startup/start-webserver.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+airflow webserver --hostname 0.0.0.0 --port 8081

backend/api/app/core/models.py

@@ -1,18 +1,36 @@
-from sqlalchemy import Column, Integer, Text, Boolean
+from sqlalchemy import Column, Integer, Text, Boolean, DateTime
 from core.database import Base
 
 
 class Team(Base):
 
     __tablename__ = 'teams'
 
-    team_id    = Column('team_id', Text, primary_key=True, index=True)
-    org_id     = Column(Text, index=True, nullable=False)
-    city       = Column(Text, nullable=False)
-    mascot     = Column(Text, nullable=False)
-    start_year = Column(Integer, nullable=False)
-    active     = Column(Boolean, nullable=False)
-    pfr_name   = Column(Text, nullable=False)
+    team_id    = Column('team_id',    Text, primary_key=True)
+    org_id     = Column('org_id',     Text)
+    city       = Column('city',       Text)
+    mascot     = Column('mascot',     Text)
+    start_year = Column('start_year', Integer)
+    active     = Column('active',     Boolean)
+    pfr_name   = Column('pft_name',   Text)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f'<Team: {self.city} {self.mascot} ({self.team_id})>'
+
+
+class Game(Base):
+
+    __tablename__ = 'games'
+
+    game_id      = Column('game_id',      Text, primary_key=True)
+    season       = Column('season',       Integer)
+    week         = Column('week',         Integer)
+    datetime     = Column('datetime',     DateTime)
+    home_team_id = Column('home_team_id', Text)
+    away_team_id = Column('away_team_id', Text)
+    home_score   = Column('home_score',   Integer)
+    away_score   = Column('away_score',   Integer)
+    has_pbp      = Column('has_pbp',      Boolean, default=False)
+
+    def __repr__(self) -> str:
+        return f'<Game: {self.game_id} | {self.away_team_id} @ {self.home_team_id}>'

backend/api/app/crud.py

@@ -1,5 +1,19 @@
 from sqlalchemy.orm import Session
-from core import models
+from sqlalchemy import insert
+from core import models, schemas
+from typing import List
+
+def add_games(db: Session, games):
+    game_objects = [models.Game(**game) for game in games]
+    db.bulk_save_objects(game_objects)
+    db.commit()
+    return games
+
+def add_teams(db: Session, teams: List[schemas.Team]):
+    team_objects = [models.Team(**team.dict()) for team in teams]
+    db.bulk_save_objects(team_objects)
+    db.commit()
+    return teams
 
 def get_teams(db: Session):
     return db.query(models.Team).all()

backend/api/app/main.py

@@ -2,11 +2,10 @@
 from fastapi.params import Depends
 from sqlalchemy.orm import Session
 from starlette.responses import RedirectResponse
-from typing import List
+from typing import List, Dict
 from core import database, schemas
 import crud
 
-
 app = FastAPI()
 
 def get_db():
@@ -16,12 +15,25 @@ def get_db():
     finally:
         db.close()
 
-@app.get('/')
-def home():
-    response = RedirectResponse(url='/redoc')
-    return response
 
 @app.get('/teams', response_model=List[schemas.Team])
 def get_teams(db: Session = Depends(get_db)):
     teams = crud.get_teams(db=db)
-    return teams
+    return teams
+
+@app.post('/teams', response_model=List[schemas.Team])
+def add_teams(teams: List[schemas.Team], db: Session = Depends(get_db)):
+    teams = crud.add_teams(db=db, teams=teams)
+    return teams
+
+@app.get('/games/past/{year}/')
+def get_past_games(year: str):
+    pass
+
+@app.get('/games/upcoming/{year}/')
+def get_upcoming_games(year: str):
+    pass
+
+@app.get('/pbp/{game_id}')
+def get_pbp(game_id: str):
+    pass

backend/docker-compose.yml

@@ -1,5 +1,8 @@
 version: '3'
 
+volumes: 
+    airflow-logs:
+
 services:
     db:
         container_name: juice-postgres
@@ -24,4 +27,52 @@ services:
         volumes: 
             - ./api/app:/app
         ports:
-            - '8000:8000'
+            - '8000:8000'
+
+    airflow-scheduler:
+        container_name: juice-airflow-scheduler
+        build:
+            context: ./airflow
+        image: juice-airflow
+        restart: always
+        ports:
+            - '8793:8793'
+        environment: 
+            AIRFLOW_USERNAME:
+            AIRFLOW_EMAIL:
+            AIRFLOW_FIRSTNAME:
+            AIRFLOW_LASTNAME:
+            AIRFLOW_PASSWORD:
+            JUICE_DB_NAME: 
+            JUICE_DB_USER:
+            JUICE_DB_PASSWORD:
+            JUICE_API_ENDPOINT: http://api:8000
+            AIRFLOW__CORE__SQL_ALCHEMY_CONN: sqlite:////opt/airflow/database/airflow.db
+        volumes:
+            - ./airflow/dags:/opt/airflow/dags
+            - ./airflow/database:/opt/airflow/database
+            - ./airflow/data:/opt/airflow/data
+            - airflow-logs:/opt/airflow/logs
+        command: ["start-scheduler.sh"]
+
+
+    airflow-webserver:
+        container_name: juice-airflow-webserver
+        build:
+            context: ./airflow
+        image: juice-airflow
+        restart: always
+        ports:
+            - '8081:8081'
+        environment: 
+            AIRFLOW__CORE__SQL_ALCHEMY_CONN: sqlite:////opt/airflow/database/airflow.db
+        volumes:
+            - ./airflow/dags:/opt/airflow/dags
+            - ./airflow/database:/opt/airflow/database
+            - airflow-logs:/opt/airflow/logs
+        command: ["start-webserver.sh"]
+        depends_on: 
+            - airflow-scheduler
+
+
+

frontend/juice/.gitignore

@@ -0,0 +1,23 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*