Fetch repositories from a GitHub or GitLab owner

src/grimoirelab/core/config/settings.py

@@ -0,0 +1,251 @@
+#
+# GrimoireLab core basic settings file.
+#
+# This file defines the required settings to run GrimoireLab.
+# Due to GrimoireLab is a Django based app, these settings are
+# based on the configuration file generated by Django by
+# default.
+#
+# Please check the next links for details about the configuration
+# in a production environment:
+#
+# https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+# https://docs.djangoproject.com/en/4.2/ref/settings/
+#
+
+import os
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+SILENCED_SYSTEM_CHECKS = [
+    'django_mysql.E016'
+]
+
+#
+# General app parameters
+#
+
+#
+# You must never enable debug in production.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#std:setting-DEBUG
+#
+
+DEBUG = os.environ.get('GRIMOIRELAB_DEBUG', 'False').lower() in ('true', '1')
+
+#
+# ALLOWED_HOST protects the site against CSRF attacks.
+# If DEBUG is set to False, you will need to configure this parameter,
+# with the host you are using to serve GrimoireLab.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#allowed-hosts
+#
+
+if 'GRIMOIRELAB_ALLOWED_HOST' in os.environ:
+    ALLOWED_HOSTS = os.environ['GRIMOIRELAB_ALLOWED_HOST'].split(',')
+else:
+    ALLOWED_HOSTS = [
+        '127.0.0.1',
+        'localhost',
+    ]
+
+#
+# The secret key must be a large random value and it must be kept secret.
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#secret-key
+#
+
+SECRET_KEY = os.environ.get('GRIMOIRELAB_SECRET_KEY', 'fake-key')
+
+#
+# Application definition - DO NOT MODIFY
+#
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django_rq',
+    'grimoirelab.core.scheduler',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'grimoirelab.core.app.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'grimoirelab.core.app.wsgi.application'
+
+
+#
+# Grimoirelab core database
+#
+# You MUST set the database parameters in order to run
+# GrimoireLab.
+#
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'HOST': os.environ.get('GRIMOIRELAB_DB_HOST', '127.0.0.1'),
+        'PORT': os.environ.get('GRIMOIRELAB_DB_PORT', 3306),
+        'USER': os.environ.get('GRIMOIRELAB_DB_USER', 'root'),
+        'PASSWORD': os.environ.get('GRIMOIRELAB_DB_PASSWORD', ''),
+        'NAME': os.environ.get('GRIMOIRELAB_DB_DATABASE', 'grimoirelab_test'),
+        'OPTIONS': {'charset': 'utf8mb4'},
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+#
+# Internationalization
+#
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+#
+#
+
+LANGUAGE_CODE = 'en-us'
+USE_I18N = True
+
+#
+# Time Zone
+#
+
+USE_TZ = True
+TIME_ZONE = 'UTC'
+
+#
+# GrimoireLab Logging
+#
+# https://docs.djangoproject.com/en/4.2/topics/logging/#configuring-logging
+#
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'simple': {
+            'format': '[{asctime}] {message}',
+            'style': '{',
+        },
+        'verbose': {
+            'format': '[{asctime} - {levelname} - {name}:{lineno}] {message}',
+            'style': '{',
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'verbose',
+        },
+    },
+    'root': {
+        'handlers': ['console'],
+        'level': 'INFO',
+    },
+}
+
+
+#
+# Static files (CSS, JavaScript, Images)
+#
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+#
+
+STATIC_URL = '/static/'
+
+
+# UI static files will be copied to the next path when
+# 'collectstatic' is run.
+# If you are serving these files in a dedicated server, you will
+# need to copy them to their final destination.
+
+STATIC_ROOT = os.path.join(BASE_DIR, 'static')
+
+
+#
+# Default primary key field type
+#
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+#
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+
+#
+# GrimoireLab uses RQ to run background and async jobs.
+# You'll HAVE TO set the next parameters in order to run
+# them in the background.
+#
+# Take into account RQ uses Redis database. You have more
+# info about these parameters on the following link:
+#
+# https://github.com/rq/django-rq
+#
+
+RQ_QUEUES = {
+    'default': {
+        'HOST': os.environ.get('GRIMOIRELAB_REDIS_HOST', '127.0.0.1'),
+        'PORT': os.environ.get('GRIMOIRELAB_REDIS_PORT', 6379),
+        'PASSWORD': os.environ.get('GRIMOIRELAB_REDIS_PASSWORD', ''),
+        'DB': os.environ.get('GRIMOIRELAB_REDIS_DB', 0),
+    },
+    'owner': {
+        'HOST': os.environ.get('GRIMOIRELAB_REDIS_HOST', '127.0.0.1'),
+        'PORT': os.environ.get('GRIMOIRELAB_REDIS_PORT', 6379),
+        'PASSWORD': os.environ.get('GRIMOIRELAB_REDIS_PASSWORD', ''),
+        'DB': os.environ.get('GRIMOIRELAB_REDIS_DB', 0),
+    }
+}
+
+#
+# Configuration for backends
+#
+
+GIT_PATH = '~/.perceval'

src/grimoirelab/core/scheduler/backends/github.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) GrimoireLab Contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+#     Jose Javier Merchante <[email protected]>
+#     Eva Millán <[email protected]>
+#
+
+import requests
+import logging
+
+GITHUB_API_URL = "https://api.github.com"
+
+logger = logging.getLogger(__name__)
+
+
+def github_owner_repositories(owner, api_token=None):
+    """Fetch GitHub repositories from an owner.
+
+    This function fetch all the repositories from a GitHub owner.
+    It return a list of repositories indicating whether the repository
+    is a fork, has issues enabled, or has pull requests enabled.
+
+    :param owner: name of the GitHub user or organization
+    :param api_token: GitHub auth token to access the API
+
+    :returns: a list of repositories including url, has_issues and fork.
+    """
+    logger.debug(
+        f"Fetching GitHub owner repositories; "
+        f"owner={owner}; ..."
+    )
+
+    headers = {}
+    if api_token:
+        headers = {'Authorization': 'token {}'.format(api_token)}
+
+    url = GITHUB_API_URL + "/users/" + owner + "/repos"
+
+    while True:
+        res = _fetch(url, headers)
+        for repo in res.json():
+            repo_data = {
+                'url': repo['html_url'],
+                'fork': repo['fork'],
+                'has_issues': repo['has_issues'],
+                'has_pull_requests': True,
+                'archived': repo['archived']
+            }
+            yield repo_data
+        if 'next' in res.links.keys():
+            url = res.links['next']['url']
+        else:
+            break
+
+    logger.info(f"GitHub owner repositories fetched; owner='{owner}'")
+
+
+def _fetch(url, headers=None):
+    """Fetch the data from a given URL.
+
+    :param url: link to the resource
+    :param headers: headers of the request
+    """
+    r = requests.get(url, headers=headers)
+    r.raise_for_status()
+    return r