Merge branch 'dev' into dev

Dockerfile.django

@@ -65,13 +65,20 @@ COPY \
   docker/entrypoint-celery-worker.sh \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-uwsgi.sh \
+  docker/entrypoint-uwsgi-cb.sh \
   docker/entrypoint-uwsgi-dev.sh \
   docker/entrypoint-uwsgi-ptvsd.sh \
   docker/entrypoint-unit-tests.sh \
   docker/entrypoint-unit-tests-devDocker.sh \
   docker/wait-for-it.sh \
   docker/certs/* \
   /
+# this file must exist. If not, then check your branch. Otherwise, it will throw an error.
+# when building, you have to be in release mode, not dev or ptvsd
+COPY \
+  docker/extra_settings/local_settings.py \
+  /app/dojo/settings/local_settings.py
+
 COPY wsgi.py manage.py docker/unit-tests.sh ./
 COPY dojo/ ./dojo/
 

docker/entrypoint-uwsgi-cb.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Allow for bind-mount setting.py overrides
+FILE=/settings/settings.py
+if test -f "$FILE"; then
+    echo "============================================================"
+    echo "     Overriding DefectDojo's settings.py with $FILE."
+    echo "============================================================"
+    cp "$FILE" /app/dojo/settings/settings.py
+fi
+
+umask 0002
+
+UWSGI_INIFILE=dojo/uwsgi.ini
+cat > $UWSGI_INIFILE<<EOF
+[uwsgi]
+$DD_UWSGI_MODE = $DD_UWSGI_ENDPOINT
+protocol = uwsgi
+module = dojo.wsgi:application
+enable-threads
+processes = ${DD_UWSGI_NUM_OF_PROCESSES:-2}
+threads = ${DD_UWSGI_NUM_OF_THREADS:-2}
+threaded-logger
+buffer-size = ${DD_UWSGI_BUFFER_SIZE:-4096}
+; HTTP endpoint is enabled for Kubernetes liveness checks. It should not be exposed as a serivce.
+http = 0.0.0.0:8081
+http-to = ${DD_UWSGI_ENDPOINT}
+EOF
+
+if [ "${DD_LOGGING_HANDLER}" = "json_console" ]; then
+    cat >> $UWSGI_INIFILE <<'EOF'
+; logging as json does not offer full tokenization for requests, everything will be in message.
+logger = stdio
+log-encoder = json {"timestamp":"${strftime:%%Y-%%m-%%d %%H:%%M:%%S%%z}", "source": "uwsgi", "message":"${msg}"}
+log-encoder = nl
+EOF
+fi
+
+exec uwsgi --ini $UWSGI_INIFILE

docker/extra_settings/local_settings.py

@@ -0,0 +1,37 @@
+# local_settings.py
+# this file will be included by settings.py *after* loading settings.dist.py
+
+from celery.schedules import crontab
+
+# add our own cb_tasks.py for tasks to get registered
+CELERY_IMPORTS += ('dojo.cb_tasks',)
+CELERY_BEAT_SCHEDULE['auto-delete-engagements'] = {
+    'task': 'dojo.cb_tasks.auto_delete_engagements',
+    'schedule': crontab(hour=9, minute=30)
+}
+
+# Temp fix - fix possible circular dups
+CELERY_BEAT_SCHEDULE['fix_loop_duplicates'] = { 
+    'task': 'dojo.tasks.fix_loop_duplicates_task',
+    'schedule': crontab(hour=9, minute=00)
+}
+
+# ensure jira status reflect on defectdojo findings
+CELERY_BEAT_SCHEDULE['jira_status_reconciliation'] = { 
+    'task': 'dojo.tasks.jira_status_reconciliation_task',
+    'schedule': timedelta(hours=24),
+    'kwargs': {'mode': 'import_status_from_jira', 'dryrun': False, 'daysback': 2}
+}
+
+# Override deduplication for certain parsers
+HASHCODE_FIELDS_PER_SCANNER['Anchore Engine Scan'] = ['title', 'severity', 'component_name', 'component_version', 'file_path']
+HASHCODE_ALLOWS_NULL_CWE['Anchore Engine Scan'] = True
+DEDUPLICATION_ALGORITHM_PER_PARSER['Anchore Engine Scan'] = DEDUPE_ALGO_HASH_CODE
+
+HASHCODE_FIELDS_PER_SCANNER['Twistlock Image Scan'] = ['title', 'severity', 'component_name', 'component_version']
+HASHCODE_ALLOWS_NULL_CWE['Twistlock Image Scan'] = True
+DEDUPLICATION_ALGORITHM_PER_PARSER['Twistlock Image Scan'] = DEDUPE_ALGO_HASH_CODE
+
+# HASHCODE_FIELDS_PER_SCANNER['Dependency Check Scan'] = ['title', 'severity', 'component_name', 'component_version']
+# HASHCODE_ALLOWS_NULL_CWE['Dependency Check Scan'] = True
+# DEDUPLICATION_ALGORITHM_PER_PARSER['Dependency Check Scan'] = DEDUPE_ALGO_HASH_CODE

docs/content/integrations/jira.md

@@ -82,13 +82,13 @@ Adding JIRA to Dojo
 
 **Customize JIRA issue description**
 
-By default Defect Dojo uses the `dojo/templates/issue-trackers/jira-description.tpl` template to render the description of the 'to be' created JIRA issue.
+By default Defect Dojo uses the `dojo/templates/issue-trackers/jira_full/jira-description.tpl` template to render the description of the 'to be' created JIRA issue.
 This file can be modified to your needs, rebuild all containers afterwards. There's also a more limited template available, which can be chosen when
 configuring a JIRA Instance or JIRA Project for a Product or Engagement:
 
 ![image](../../images/jira_issue_templates.png)
 
-Any template add to  `dojo/templates/issue-trackers/` will be added to the dropdown (after rebuilding/restarting the containers).
+Any folder added to  `dojo/templates/issue-trackers/` will be added to the dropdown (after rebuilding/restarting the containers).
 
 Engagement Epic Mapping
 .......................

docs/content/running/upgrading.md

@@ -122,6 +122,12 @@ Note that the below fields are now optional without default value. They will not
 - url
 
 
+Upgrading to DefectDojo Version 1.15.x
+--------------------------------------
+- See release notes: https://github.com/DefectDojo/django-DefectDojo/releases/tag/1.13.0
+- If you have made changes to JIRA templates or the template config in the JIRA Project config for instances/products/engagements:
+The jira template settings introduced in 1.13 have been changed. You now have to select a subfolder instead of a sinlge template file. If you have chosen a non-default template here, you have to reapply that to all products / engagements. Also you have to move your custom templates into the correct subfolder in `dojo/templates/issue-trackers/`.
+
 
 Upgrading to DefectDojo Version 1.13.x
 --------------------------------------

dojo/api_v2/serializers.py

@@ -1,5 +1,5 @@
 from drf_yasg.utils import swagger_serializer_method
-from dojo.models import Product, Engagement, Test, Finding, \
+from dojo.models import Finding_Group, Product, Engagement, Test, Finding, \
     User, Stub_Finding, Risk_Acceptance, \
     Finding_Template, Test_Type, Development_Environment, NoteHistory, \
     JIRA_Issue, Tool_Product_Settings, Tool_Configuration, Tool_Type, \
@@ -650,9 +650,18 @@ class Meta:
         fields = '__all__'
 
 
+class FindingGroupSerializer(serializers.ModelSerializer):
+    jira_issue = JIRAIssueSerializer(read_only=True)
+
+    class Meta:
+        model = Finding_Group
+        fields = ('name', 'test', 'jira_issue')
+
+
 class TestSerializer(TaggitSerializer, serializers.ModelSerializer):
     tags = TagListSerializerField(required=False)
     test_type_name = serializers.ReadOnlyField()
+    finding_groups = FindingGroupSerializer(source='finding_group_set', many=True, read_only=True)
 
     class Meta:
         model = Test
@@ -811,6 +820,7 @@ class FindingSerializer(TaggitSerializer, serializers.ModelSerializer):
     jira_creation = serializers.SerializerMethodField(read_only=True)
     jira_change = serializers.SerializerMethodField(read_only=True)
     display_status = serializers.SerializerMethodField()
+    finding_groups = FindingGroupSerializer(source='finding_group_set', many=True, read_only=True)
 
     class Meta:
         model = Finding

dojo/api_v2/views.py

@@ -476,6 +476,8 @@ def notes(self, request, pk=None):
 
             if finding.has_jira_issue:
                 jira_helper.add_comment(finding, note)
+            elif finding.has_jira_group_issue:
+                jira_helper.add_comment(finding.finding_group, note)
 
             serialized_note = serializers.NoteSerializer({
                 "author": author, "entry": entry,

dojo/cb_tasks.py

@@ -0,0 +1,17 @@
+from dojo.cb_utils import auto_delete_engagements
+# from dojo.models import System_Settings
+from dojo.celery import app
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+
+@app.task(name='dojo.cb_tasks.auto_delete_engagements')
+def async_auto_delete_engagements(*args, **kwargs):
+    try:
+        # system_settings = System_Settings.objects.get()
+        # if system_settings.engagement_auto_delete_enable:
+        logger.info("Automatically deleting engagements and related as needed")
+        auto_delete_engagements(*args, **kwargs)
+    except Exception as e:
+        logger.error("An unexpected error was thrown calling the engagements auto deletion code: {}".format(e))

dojo/cb_utils.py

@@ -0,0 +1,71 @@
+from dojo.models import Finding, Engagement, System_Settings
+import logging
+from datetime import datetime, timedelta
+from django.utils import timezone
+from django.db.models import Q, Exists, OuterRef
+
+
+logger = logging.getLogger(__name__)
+
+
+def auto_delete_engagements():
+    # TODO implement dry-run option
+    """
+    For an engagement to be in-scope for automated deletion, the following rules apply:
+    - must have been updated before x days (as defined in system settings)
+    - (hardcoded) must be a CI/CD engagement
+    - (hardcoded) must only contain duplicate findings
+    - (hardocded) must not contain any notes on any of its findings
+
+    The original use-case of this feature relates to the mass imports that one can have through CI pipelines,
+    generating a vast amount of findings which ultimately will boggle down defectdojo's performance
+    and make it harder to see what needs to be seen.
+    """
+
+    """
+    def _notify(engagement_id, engagement_title):
+        create_notification(
+            event='auto_delete_engagement',
+            title=engagement_title,
+            id=engagement_id,
+        )
+    """
+
+    system_settings = System_Settings.objects.get()
+    # if system_settings.engagement_auto_delete_enable:
+    # how to not exclude the tag when not empty? If empty, then query results are unexpected.
+    # setting arbitrary string for now, which is unlikely to be a used tag.
+    # lock_tag = system_settings.engagement_auto_delete_lock_tag or 'qAEH2HL6Qd9ofZYLCGykN2WQ'
+    lock_tag = 'donotdelete'
+    logger.info("Proceeding with automatic engagements deletion, for engagements older than {} days".format(
+        30
+    ))
+    logger.info("Lock tag is {}".format(lock_tag))
+
+    # cutoff_date = timezone.make_aware(datetime.today()) - timedelta(days=system_settings.engagement_auto_delete_days)
+    cutoff_date = timezone.make_aware(datetime.today()) - timedelta(days=30)
+    cutoff_date.tzinfo
+    logger.info("Cutoff date is {}".format(cutoff_date))
+    engagements_to_delete = Engagement.objects.annotate(
+        all_duplicates=~Exists(
+            Finding.objects.filter(~Q(duplicate=True), test__engagement_id=OuterRef('pk'))
+        ),
+        has_no_note=~Exists(
+            Finding.objects.filter(~Q(notes__isnull=True), test__engagement_id=OuterRef('pk'))
+        ),
+    ).filter(
+        engagement_type='CI/CD',
+        created__lt=cutoff_date,
+        all_duplicates=True,
+        has_no_note=True
+    ).exclude(
+        tags__name__contains=lock_tag
+    )
+
+    for engagement in engagements_to_delete:
+        logger.info("Deleting engagement id {} ({})".format(engagement.id, engagement.name))
+        # _notify(engagement, "Engagement {} ({})- auto-deleted".format(engagement.id, engagement.name))
+        engagement.delete()
+
+    else:
+        logger.debug("Automatic engagement deletion is not activated.")

dojo/db_migrations/0080_jira_issue_templates.py

@@ -13,11 +13,11 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='jira_instance',
             name='issue_template',
-            field=models.CharField(blank=True, help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.', max_length=255, null=True),
+            field=models.CharField(blank=True, help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.', max_length=255, null=True),
         ),
         migrations.AddField(
             model_name='jira_project',
             name='issue_template',
-            field=models.CharField(blank=True, help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.', max_length=255, null=True),
+            field=models.CharField(blank=True, help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.', max_length=255, null=True),
         ),
     ]

dojo/db_migrations/0085_finding_groups.py

@@ -0,0 +1,45 @@
+# Generated by Django 2.2.17 on 2021-03-21 07:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0084_add_extras_in_tool'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='jira_instance',
+            old_name='issue_template',
+            new_name='issue_template_dir',
+        ),
+        migrations.RenameField(
+            model_name='jira_project',
+            old_name='issue_template',
+            new_name='issue_template_dir',
+        ),
+        migrations.CreateModel(
+            name='Finding_Group',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('name', models.CharField(max_length=255)),
+                ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dojo.Dojo_User')),
+                ('findings', models.ManyToManyField(to='dojo.Finding')),
+                ('test', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dojo.Test')),
+            ],
+            options={
+                'ordering': ['id'],
+            },
+        ),
+        migrations.AddField(
+            model_name='jira_issue',
+            name='finding_group',
+            field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='dojo.Finding_Group'),
+        ),
+    ]

dojo/decorators.py

@@ -48,7 +48,7 @@ def dojo_model_to_id(_func=None, *, parameter=0):
     # logger.debug('dec_kwargs:' + str(dec_kwargs))
     # logger.debug('_func:%s', _func)
 
-    def dojo_model_from_id_internal(func, *args, **kwargs):
+    def dojo_model_to_id_internal(func, *args, **kwargs):
         @wraps(func)
         def __wrapper__(*args, **kwargs):
             if not settings.CELERY_PASS_MODEL_BY_ID:
@@ -57,7 +57,7 @@ def __wrapper__(*args, **kwargs):
             model_or_id = get_parameter_froms_args_kwargs(args, kwargs, parameter)
 
             if model_or_id:
-                if isinstance(model_or_id, models.Model) and we_want_async():
+                if isinstance(model_or_id, models.Model) and we_want_async(*args, **kwargs):
                     logger.debug('converting model_or_id to id: %s', model_or_id)
                     id = model_or_id.id
                     args = list(args)
@@ -69,9 +69,9 @@ def __wrapper__(*args, **kwargs):
 
     if _func is None:
         # decorator called without parameters
-        return dojo_model_from_id_internal
+        return dojo_model_to_id_internal
     else:
-        return dojo_model_from_id_internal(_func)
+        return dojo_model_to_id_internal(_func)
 
 
 # decorator with parameters needs another wrapper layer
@@ -96,7 +96,7 @@ def __wrapper__(*args, **kwargs):
             model_or_id = get_parameter_froms_args_kwargs(args, kwargs, parameter)
 
             if model_or_id:
-                if not isinstance(model_or_id, models.Model) and we_want_async():
+                if not isinstance(model_or_id, models.Model) and we_want_async(*args, **kwargs):
                     logger.debug('instantiating model_or_id: %s for model: %s', model_or_id, model)
                     try:
                         instance = model.objects.get(id=model_or_id)