Merge pull request #75 from SentiStock/dev

final changes

.env-example

@@ -4,8 +4,9 @@ DEBUG=True
 #IMAGE_SOURCE is "local", "remote" or "private". It indicates from where to pull the containers.
 IMAGE_SOURCE=local
 
-#DATABASE_ENV indicates which databse you are using "local" or "production", the only difference between these is that "local" fetches congiguration from default settings and production uses config specified here
+#Indicates which databse you are using "local" or "production", the only difference between these is that "local" fetches congiguration from default settings and production uses config specified here
 DATABASE_ENV=local
+REDIS_ENV=local
 
 #ALLOWED_HOSTS should be separated by comma in a form ALLOWED_HOSTS=example.com,nextexample.com, if you want to allow all put ALLOWED_HOSTS=*
 ALLOWED_HOSTS=None
@@ -17,4 +18,10 @@ DATABASE_PORT=None
 DATABASE_USERNAME=None
 DATABASE_PASSWORD=None
 
-TWITTER_BEARER_TOKEN=None
+REDIS_HOST=None
+REDIS_PORT=None
+
+TWITTER_BEARER_TOKEN=None
+
+SENTITWEETAPI_SENTIMENT_X_FUNCTIONS_KEY=None
+SENTITWEETAPI_SENTIMENT_URL=None

.gitignore

@@ -14,11 +14,16 @@ lib/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+notebooks/
 
 # Environmental vars
 .env
 .env-bak
 
+.env-remote
+.env-local
+.env-production
+
 #Environments
 .venv
 venv/

Makefile

@@ -51,5 +51,5 @@ db-backup:
 	@$(docker_db) pg_dump -U postgres dbsentitweet > db_backup.sql
 
 .PHONY: install-requirements
-install: # Execute migrate command in sentitweet container
+install-requirements: # Execute migrate command in sentitweet container
 	@$(docker_web) pip install -r /app/requirements.txt

apps/authentication/migrations/0002_alter_favorite_user.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2.16 on 2022-12-10 16:36
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('authentication', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='favorite',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='my_favorites', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

apps/authentication/migrations/0003_alter_favorite_unique_together.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2022-12-11 15:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('authentication', '0002_alter_favorite_user'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='favorite',
+            unique_together={('user', 'favorite_ct', 'favorite_id')},
+        ),
+    ]

apps/authentication/models.py

@@ -4,9 +4,10 @@
 from django.db import models
 from django.db.models import Q
 
+from home.models import PandasModelMixin
 
 class Favorite(models.Model):
-    user = models.ForeignKey('authentication.Contributor', related_name='favorites', on_delete=models.CASCADE)
+    user = models.ForeignKey('authentication.Contributor', related_name='my_favorites', on_delete=models.CASCADE)
     favorite_ct = models.ForeignKey(
         ContentType, 
         related_name='favorite_obj', 
@@ -21,10 +22,11 @@ def __str__(self):
         return f'{self.user} follows {self.favorite}'
 
     class Meta:
+        unique_together = ['user', 'favorite_ct', 'favorite_id']
         ordering = ['favorite_ct']
 
 
-class FavoritesModelMixin(models.Model):
+class FavoritesModelMixin(PandasModelMixin):
     @property
     def favorites(self):
         return Favorite.objects.filter(
@@ -46,3 +48,8 @@ class Contributor(AbstractUser, FavoritesModelMixin):
         default = 'private'
     )
 
+    def is_favorite(self, object):
+        return self.my_favorites.filter(
+            Q(favorite_id=object.id) & Q(favorite_ct__model=object._meta.model_name)
+        ).exists()
+

apps/home/models.py

@@ -6,5 +6,25 @@
 from django.db import models
 from django.contrib.auth.models import User
 
-# Create your models here.
+import pandas as pd
+
+class PandasModelMixin(models.Model):
+    @classmethod
+    def as_dataframe(cls, queryset=None, field_list=None):
+        if queryset is None:
+            queryset = cls.objects.all()
+        if field_list is None:
+            field_list = [_field.name for _field in cls._meta._get_fields(reverse=False)]
+
+        data = []
+        [data.append([obj.serializable_value(column) for column in field_list]) for obj in queryset]
+
+        columns = field_list
+
+        df = pd.DataFrame(data, columns=columns)
+        return df
+
+    class Meta:
+        abstract = True
+
 

apps/home/views.py

@@ -10,13 +10,13 @@
 from django.template import loader
 from django.urls import reverse
 from stock.models import Company
-from stock.utils import get_relevent_model_context
+from stock.utils import get_relevant_model_context
 from tweet.models import HashTag
 
 
 @login_required(login_url="/login/")
 def index(request):
-    context = get_relevent_model_context()
+    context = get_relevant_model_context(companies=True, sets=True, users=True, contributors=True, hashtags=True)
     context['segment'] = 'index'
 
     html_template = loader.get_template('home/index.html')
@@ -27,7 +27,7 @@ def index(request):
 def pages(request):
     # All resource paths end in .html.
     # Pick out the html file name from the url. And load that template.
-    context = get_relevent_model_context()
+    context = get_relevant_model_context(companies=True, sets=True, users=True, contributors=True, hashtags=True)
 
     try:
         load_template = request.path.split('/')[-1]

apps/stock/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.1.3 on 2022-12-07 20:55
+# Generated by Django 3.2.16 on 2022-12-10 15:33
 
 from django.db import migrations, models
 
@@ -17,7 +17,7 @@ class Migration(migrations.Migration):
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=255)),
                 ('symbol', models.CharField(max_length=31)),
-                ('market_cap', models.CharField(blank=True, max_length=63, null=True)),
+                ('market_cap', models.BigIntegerField(blank=True, null=True)),
                 ('stock_price', models.IntegerField(blank=True, null=True)),
                 ('country', models.CharField(blank=True, max_length=63, null=True)),
                 ('twitter_query_set', models.CharField(blank=True, max_length=255, null=True)),

apps/stock/models.py

@@ -1,8 +1,9 @@
 import datetime
 
 from authentication.models import FavoritesModelMixin
+from django.conf import settings
 from django.db import models
-from django.db.models import Count, Q, Sum
+from django.db.models import Avg, Count, Q, Sum
 from django.utils import timezone
 from tweet import models as tweet_models
 
@@ -35,11 +36,43 @@ def get_search_hashtags(self, top=10):
 
     def get_top_hashtags(self, top=10):
         return self.hashtags.annotate(t_count=Count('tweets')).order_by('-t_count')[:top]
+
+    @property
+    def tweet_count(self):
+        return self.tweets.count()
+
+    @property
+    def hashtag_count(self):
+        return self.hashtags.count()
+
+    @property
+    def twitter_user_count(self):
+        return self.twitter_users.count()
+
+    @property
+    def favorite_count(self):
+        return self.favorites.count()
 
     @property
     def search_name(self):
         return self.name.split(' ')[0].split('.')[0]
 
+    @property
+    def tweet_count(self):
+        return self.tweets.count()
+
+    @property
+    def hashtag_count(self):
+        return self.hashtags.count()
+
+    @property
+    def twitter_user_count(self):
+        return self.twitter_users.count()
+
+    @property
+    def favorite_count(self):
+        return self.favorites.count()
+
     @property
     def newest_tweet(self):
         return self.tweets.order_by('-post_date').first()
@@ -50,7 +83,7 @@ def oldest_tweet(self):
 
     @property
     def is_up_to_date(self):
-        return self.newest_tweet.post_date > (timezone.now() - datetime.timedelta(7)) if self.newest_tweet else False
+        return self.newest_tweet.post_date > (timezone.now() - datetime.timedelta(settings.DAYS_TILL_TWEETS_ARE_OUTDATED)) if self.newest_tweet else False
 
     @property
     def twitter_users(self):
@@ -64,7 +97,6 @@ def top_twitter_users(self, top=10):
 
     @property
     def total_likes(self):
-        print(self.tweets.aggregate(Sum('like_number')))
         return self.tweets.aggregate(Sum('like_number'))['like_number__sum']
 
     @property
@@ -73,4 +105,43 @@ def total_retweets(self):
 
     @property
     def total_comments(self):
-        return self.tweets.aggregate(Sum('comment_number'))['comment_number__sum']
+        return self.tweets.aggregate(Sum('comment_number'))['comment_number__sum']
+
+    def get_tweets(self, from_date_time=None, till_date_time=None):
+        if not from_date_time:
+            from_date_time = self.oldest_tweet.post_date if self.oldest_tweet else timezone.now()
+        if not till_date_time:
+            till_date_time = self.newest_tweet.post_date if self.newest_tweet else timezone.now()
+        return self.tweets.filter(
+            Q(post_date__gte=from_date_time) & Q(post_date__lte=till_date_time)
+        )
+
+    def get_compound(self, from_date_time=None, till_date_time=None):
+        tweets = self.get_tweets(from_date_time, till_date_time)
+        compound = tweets.aggregate(Avg('sentiment_compound'))['sentiment_compound__avg']
+        if compound:
+            return round(compound, 2)
+        return 0
+
+    def get_sentiment_label(self, from_date_time=None, till_date_time=None):
+        compound = self.get_compound(from_date_time, till_date_time)
+        if compound > settings.SENTIMENT_COMPOUND_TRHESHOLD:
+            return 'Positive'
+        if compound < settings.SENTIMENT_COMPOUND_TRHESHOLD * -1:
+            return 'Negative'
+        return 'Neutral'
+
+    def get_hashtags(self, from_date_time=None, till_date_time=None):
+        tweets = self.get_tweets(from_date_time, till_date_time)
+        hashtag_ids = set(tweets.values_list('hashtags', flat=True).distinct())
+        return self.hashtags.filter(id__in=hashtag_ids)
+
+    def get_twitter_users(self, from_date_time=None, till_date_time=None):
+        tweets = self.get_tweets(from_date_time, till_date_time)
+        user_ids = set(tweets.values_list('user', flat=True).distinct())
+        return tweet_models.TwitterUser.objects.filter(id__in=user_ids)
+
+    def get_hashtags(self, from_date_time=None, till_date_time=None):
+        tweets = self.get_tweets(from_date_time, till_date_time)
+        hashtag_ids = set(tweets.values_list('hashtags', flat=True).distinct())
+        return tweet_models.HashTag.objects.filter(id__in=hashtag_ids)

apps/stock/urls.py

@@ -1,7 +1,11 @@
 from django.urls import path
-from stock.views import company, company_detail
+from stock.views import company, company_detail, discover, trends
 
 urlpatterns = [
+
+    path('discover', discover, name='discover'),
+	path('trends', trends, name='trends'),
 	path('companies/', company, name='comapny'),
 	path('companies/<str:company_symbol_or_name>/', company_detail, name='comapany_detail'),
 ]
+

apps/stock/utils.py

@@ -1,6 +1,6 @@
 from django.db.models import Count, Q
 from stock.models import Company
-from tweet.models import HashTag, Set, TwitterUser
+from tweet.models import Contributor, HashTag, Set, TwitterUser
 from tweet.utils import cluster_tweets
 
 
@@ -22,19 +22,27 @@ def get_company_by_symbol_or_name(symbol_or_name):
     return None
 
 
-def get_relevent_model_context():
-    companies = Company.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count')
-    hashtags = HashTag.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count')
-    users = TwitterUser.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count')
-    sets = Set.objects.filter(mode='public')#.order_by('followers')
+def get_relevant_model_context(companies=False, hashtags=False, sets=False, users=False, contributors=False):
 
-    return {'sets': sets, 'companies': companies, 'hashtags': hashtags, 'users': users}
+    companies_objects = Company.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count') if companies else dict()
+    hashtags_objects = HashTag.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count') if hashtags else dict()
+    users_objects = TwitterUser.objects.all().annotate(t_count=Count('tweets')).order_by('-t_count') if users else dict()
+    sets_objects = Set.objects.filter(mode='public') if sets else dict()
+    contributors_objects = Contributor.objects.all() if contributors else dict()
+
+    return {
+        'sets': sets_objects, 
+        'companies': companies_objects,
+        'hashtags': hashtags_objects,
+        'users': users_objects,
+        'contributors': contributors_objects
+    }
 
 
 def get_cluster_context(tweets):
     if len(tweets) > 50:
-        best_tweets, top_words = cluster_tweets(tweets, number_of_best_tweets=5)
+        best_tweets, top_words, info = cluster_tweets(tweets, number_of_best_tweets=5)
         # TODO make best_tweets to be all attributes
-        return list(zip(top_words.values(), best_tweets.groupby('cluster').text.apply(list)))
+        return zip(top_words.values(), best_tweets.groupby('cluster').text.apply(list), info.to_dict('index').values())
     return None