diff --git a/Dockerfile.django b/Dockerfile.django index b4d13503163..db83a32aec9 100644 --- a/Dockerfile.django +++ b/Dockerfile.django @@ -65,6 +65,7 @@ COPY \ docker/entrypoint-celery-worker.sh \ docker/entrypoint-initializer.sh \ docker/entrypoint-uwsgi.sh \ + docker/entrypoint-uwsgi-cb.sh \ docker/entrypoint-uwsgi-dev.sh \ docker/entrypoint-uwsgi-ptvsd.sh \ docker/entrypoint-unit-tests.sh \ @@ -72,6 +73,12 @@ COPY \ docker/wait-for-it.sh \ docker/certs/* \ / +# this file must exist. If not, then check your branch. Otherwise, it will throw an error. +# when building, you have to be in release mode, not dev or ptvsd +COPY \ + docker/extra_settings/local_settings.py \ + /app/dojo/settings/local_settings.py + COPY wsgi.py manage.py docker/unit-tests.sh ./ COPY dojo/ ./dojo/ diff --git a/docker/entrypoint-uwsgi-cb.sh b/docker/entrypoint-uwsgi-cb.sh new file mode 100755 index 00000000000..043cd2eac19 --- /dev/null +++ b/docker/entrypoint-uwsgi-cb.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +# Allow for bind-mount setting.py overrides +FILE=/settings/settings.py +if test -f "$FILE"; then + echo "============================================================" + echo " Overriding DefectDojo's settings.py with $FILE." + echo "============================================================" + cp "$FILE" /app/dojo/settings/settings.py +fi + +umask 0002 + +UWSGI_INIFILE=dojo/uwsgi.ini +cat > $UWSGI_INIFILE<> $UWSGI_INIFILE <<'EOF' +; logging as json does not offer full tokenization for requests, everything will be in message. +logger = stdio +log-encoder = json {"timestamp":"${strftime:%%Y-%%m-%%d %%H:%%M:%%S%%z}", "source": "uwsgi", "message":"${msg}"} +log-encoder = nl +EOF +fi + +exec uwsgi --ini $UWSGI_INIFILE diff --git a/docker/extra_settings/local_settings.py b/docker/extra_settings/local_settings.py new file mode 100644 index 00000000000..4e979b198e3 --- /dev/null +++ b/docker/extra_settings/local_settings.py @@ -0,0 +1,37 @@ +# local_settings.py +# this file will be included by settings.py *after* loading settings.dist.py + +from celery.schedules import crontab + +# add our own cb_tasks.py for tasks to get registered +CELERY_IMPORTS += ('dojo.cb_tasks',) +CELERY_BEAT_SCHEDULE['auto-delete-engagements'] = { + 'task': 'dojo.cb_tasks.auto_delete_engagements', + 'schedule': crontab(hour=9, minute=30) +} + +# Temp fix - fix possible circular dups +CELERY_BEAT_SCHEDULE['fix_loop_duplicates'] = { + 'task': 'dojo.tasks.fix_loop_duplicates_task', + 'schedule': crontab(hour=9, minute=00) +} + +# ensure jira status reflect on defectdojo findings +CELERY_BEAT_SCHEDULE['jira_status_reconciliation'] = { + 'task': 'dojo.tasks.jira_status_reconciliation_task', + 'schedule': timedelta(hours=24), + 'kwargs': {'mode': 'import_status_from_jira', 'dryrun': False, 'daysback': 2} +} + +# Override deduplication for certain parsers +HASHCODE_FIELDS_PER_SCANNER['Anchore Engine Scan'] = ['title', 'severity', 'component_name', 'component_version', 'file_path'] +HASHCODE_ALLOWS_NULL_CWE['Anchore Engine Scan'] = True +DEDUPLICATION_ALGORITHM_PER_PARSER['Anchore Engine Scan'] = DEDUPE_ALGO_HASH_CODE + +HASHCODE_FIELDS_PER_SCANNER['Twistlock Image Scan'] = ['title', 'severity', 'component_name', 'component_version'] +HASHCODE_ALLOWS_NULL_CWE['Twistlock Image Scan'] = True +DEDUPLICATION_ALGORITHM_PER_PARSER['Twistlock Image Scan'] = DEDUPE_ALGO_HASH_CODE + +# HASHCODE_FIELDS_PER_SCANNER['Dependency Check Scan'] = ['title', 'severity', 'component_name', 'component_version'] +# HASHCODE_ALLOWS_NULL_CWE['Dependency Check Scan'] = True +# DEDUPLICATION_ALGORITHM_PER_PARSER['Dependency Check Scan'] = DEDUPE_ALGO_HASH_CODE diff --git a/docs/content/integrations/jira.md b/docs/content/integrations/jira.md index 82f20f63df6..047fd52a5c9 100644 --- a/docs/content/integrations/jira.md +++ b/docs/content/integrations/jira.md @@ -82,13 +82,13 @@ Adding JIRA to Dojo **Customize JIRA issue description** -By default Defect Dojo uses the `dojo/templates/issue-trackers/jira-description.tpl` template to render the description of the 'to be' created JIRA issue. +By default Defect Dojo uses the `dojo/templates/issue-trackers/jira_full/jira-description.tpl` template to render the description of the 'to be' created JIRA issue. This file can be modified to your needs, rebuild all containers afterwards. There's also a more limited template available, which can be chosen when configuring a JIRA Instance or JIRA Project for a Product or Engagement: ![image](../../images/jira_issue_templates.png) -Any template add to `dojo/templates/issue-trackers/` will be added to the dropdown (after rebuilding/restarting the containers). +Any folder added to `dojo/templates/issue-trackers/` will be added to the dropdown (after rebuilding/restarting the containers). Engagement Epic Mapping ....................... diff --git a/docs/content/running/upgrading.md b/docs/content/running/upgrading.md index 8471e7318e5..2ee9fe37e62 100644 --- a/docs/content/running/upgrading.md +++ b/docs/content/running/upgrading.md @@ -122,6 +122,12 @@ Note that the below fields are now optional without default value. They will not - url +Upgrading to DefectDojo Version 1.15.x +-------------------------------------- +- See release notes: https://github.com/DefectDojo/django-DefectDojo/releases/tag/1.13.0 +- If you have made changes to JIRA templates or the template config in the JIRA Project config for instances/products/engagements: +The jira template settings introduced in 1.13 have been changed. You now have to select a subfolder instead of a sinlge template file. If you have chosen a non-default template here, you have to reapply that to all products / engagements. Also you have to move your custom templates into the correct subfolder in `dojo/templates/issue-trackers/`. + Upgrading to DefectDojo Version 1.13.x -------------------------------------- diff --git a/dojo/api_v2/serializers.py b/dojo/api_v2/serializers.py index 57cb89e87a1..d5d8a6e524a 100644 --- a/dojo/api_v2/serializers.py +++ b/dojo/api_v2/serializers.py @@ -1,5 +1,5 @@ from drf_yasg.utils import swagger_serializer_method -from dojo.models import Product, Engagement, Test, Finding, \ +from dojo.models import Finding_Group, Product, Engagement, Test, Finding, \ User, Stub_Finding, Risk_Acceptance, \ Finding_Template, Test_Type, Development_Environment, NoteHistory, \ JIRA_Issue, Tool_Product_Settings, Tool_Configuration, Tool_Type, \ @@ -650,9 +650,18 @@ class Meta: fields = '__all__' +class FindingGroupSerializer(serializers.ModelSerializer): + jira_issue = JIRAIssueSerializer(read_only=True) + + class Meta: + model = Finding_Group + fields = ('name', 'test', 'jira_issue') + + class TestSerializer(TaggitSerializer, serializers.ModelSerializer): tags = TagListSerializerField(required=False) test_type_name = serializers.ReadOnlyField() + finding_groups = FindingGroupSerializer(source='finding_group_set', many=True, read_only=True) class Meta: model = Test @@ -811,6 +820,7 @@ class FindingSerializer(TaggitSerializer, serializers.ModelSerializer): jira_creation = serializers.SerializerMethodField(read_only=True) jira_change = serializers.SerializerMethodField(read_only=True) display_status = serializers.SerializerMethodField() + finding_groups = FindingGroupSerializer(source='finding_group_set', many=True, read_only=True) class Meta: model = Finding diff --git a/dojo/api_v2/views.py b/dojo/api_v2/views.py index 191ff5f171d..8b2947ce7dd 100644 --- a/dojo/api_v2/views.py +++ b/dojo/api_v2/views.py @@ -476,6 +476,8 @@ def notes(self, request, pk=None): if finding.has_jira_issue: jira_helper.add_comment(finding, note) + elif finding.has_jira_group_issue: + jira_helper.add_comment(finding.finding_group, note) serialized_note = serializers.NoteSerializer({ "author": author, "entry": entry, diff --git a/dojo/cb_tasks.py b/dojo/cb_tasks.py new file mode 100644 index 00000000000..8cbaa36b01e --- /dev/null +++ b/dojo/cb_tasks.py @@ -0,0 +1,17 @@ +from dojo.cb_utils import auto_delete_engagements +# from dojo.models import System_Settings +from dojo.celery import app +from celery.utils.log import get_task_logger + +logger = get_task_logger(__name__) + + +@app.task(name='dojo.cb_tasks.auto_delete_engagements') +def async_auto_delete_engagements(*args, **kwargs): + try: + # system_settings = System_Settings.objects.get() + # if system_settings.engagement_auto_delete_enable: + logger.info("Automatically deleting engagements and related as needed") + auto_delete_engagements(*args, **kwargs) + except Exception as e: + logger.error("An unexpected error was thrown calling the engagements auto deletion code: {}".format(e)) diff --git a/dojo/cb_utils.py b/dojo/cb_utils.py new file mode 100644 index 00000000000..2ef7697db83 --- /dev/null +++ b/dojo/cb_utils.py @@ -0,0 +1,71 @@ +from dojo.models import Finding, Engagement, System_Settings +import logging +from datetime import datetime, timedelta +from django.utils import timezone +from django.db.models import Q, Exists, OuterRef + + +logger = logging.getLogger(__name__) + + +def auto_delete_engagements(): + # TODO implement dry-run option + """ + For an engagement to be in-scope for automated deletion, the following rules apply: + - must have been updated before x days (as defined in system settings) + - (hardcoded) must be a CI/CD engagement + - (hardcoded) must only contain duplicate findings + - (hardocded) must not contain any notes on any of its findings + + The original use-case of this feature relates to the mass imports that one can have through CI pipelines, + generating a vast amount of findings which ultimately will boggle down defectdojo's performance + and make it harder to see what needs to be seen. + """ + + """ + def _notify(engagement_id, engagement_title): + create_notification( + event='auto_delete_engagement', + title=engagement_title, + id=engagement_id, + ) + """ + + system_settings = System_Settings.objects.get() + # if system_settings.engagement_auto_delete_enable: + # how to not exclude the tag when not empty? If empty, then query results are unexpected. + # setting arbitrary string for now, which is unlikely to be a used tag. + # lock_tag = system_settings.engagement_auto_delete_lock_tag or 'qAEH2HL6Qd9ofZYLCGykN2WQ' + lock_tag = 'donotdelete' + logger.info("Proceeding with automatic engagements deletion, for engagements older than {} days".format( + 30 + )) + logger.info("Lock tag is {}".format(lock_tag)) + + # cutoff_date = timezone.make_aware(datetime.today()) - timedelta(days=system_settings.engagement_auto_delete_days) + cutoff_date = timezone.make_aware(datetime.today()) - timedelta(days=30) + cutoff_date.tzinfo + logger.info("Cutoff date is {}".format(cutoff_date)) + engagements_to_delete = Engagement.objects.annotate( + all_duplicates=~Exists( + Finding.objects.filter(~Q(duplicate=True), test__engagement_id=OuterRef('pk')) + ), + has_no_note=~Exists( + Finding.objects.filter(~Q(notes__isnull=True), test__engagement_id=OuterRef('pk')) + ), + ).filter( + engagement_type='CI/CD', + created__lt=cutoff_date, + all_duplicates=True, + has_no_note=True + ).exclude( + tags__name__contains=lock_tag + ) + + for engagement in engagements_to_delete: + logger.info("Deleting engagement id {} ({})".format(engagement.id, engagement.name)) + # _notify(engagement, "Engagement {} ({})- auto-deleted".format(engagement.id, engagement.name)) + engagement.delete() + + else: + logger.debug("Automatic engagement deletion is not activated.") diff --git a/dojo/db_migrations/0080_jira_issue_templates.py b/dojo/db_migrations/0080_jira_issue_templates.py index 36c1fb6590c..617a4b19ed9 100644 --- a/dojo/db_migrations/0080_jira_issue_templates.py +++ b/dojo/db_migrations/0080_jira_issue_templates.py @@ -13,11 +13,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='jira_instance', name='issue_template', - field=models.CharField(blank=True, help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.', max_length=255, null=True), + field=models.CharField(blank=True, help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.', max_length=255, null=True), ), migrations.AddField( model_name='jira_project', name='issue_template', - field=models.CharField(blank=True, help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.', max_length=255, null=True), + field=models.CharField(blank=True, help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.', max_length=255, null=True), ), ] diff --git a/dojo/db_migrations/0085_finding_groups.py b/dojo/db_migrations/0085_finding_groups.py new file mode 100644 index 00000000000..ef3009b580f --- /dev/null +++ b/dojo/db_migrations/0085_finding_groups.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.17 on 2021-03-21 07:58 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0084_add_extras_in_tool'), + ] + + operations = [ + migrations.RenameField( + model_name='jira_instance', + old_name='issue_template', + new_name='issue_template_dir', + ), + migrations.RenameField( + model_name='jira_project', + old_name='issue_template', + new_name='issue_template_dir', + ), + migrations.CreateModel( + name='Finding_Group', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), + ('name', models.CharField(max_length=255)), + ('creator', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dojo.Dojo_User')), + ('findings', models.ManyToManyField(to='dojo.Finding')), + ('test', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dojo.Test')), + ], + options={ + 'ordering': ['id'], + }, + ), + migrations.AddField( + model_name='jira_issue', + name='finding_group', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='dojo.Finding_Group'), + ), + ] diff --git a/dojo/decorators.py b/dojo/decorators.py index d37e6586d87..4f7b1707527 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -48,7 +48,7 @@ def dojo_model_to_id(_func=None, *, parameter=0): # logger.debug('dec_kwargs:' + str(dec_kwargs)) # logger.debug('_func:%s', _func) - def dojo_model_from_id_internal(func, *args, **kwargs): + def dojo_model_to_id_internal(func, *args, **kwargs): @wraps(func) def __wrapper__(*args, **kwargs): if not settings.CELERY_PASS_MODEL_BY_ID: @@ -57,7 +57,7 @@ def __wrapper__(*args, **kwargs): model_or_id = get_parameter_froms_args_kwargs(args, kwargs, parameter) if model_or_id: - if isinstance(model_or_id, models.Model) and we_want_async(): + if isinstance(model_or_id, models.Model) and we_want_async(*args, **kwargs): logger.debug('converting model_or_id to id: %s', model_or_id) id = model_or_id.id args = list(args) @@ -69,9 +69,9 @@ def __wrapper__(*args, **kwargs): if _func is None: # decorator called without parameters - return dojo_model_from_id_internal + return dojo_model_to_id_internal else: - return dojo_model_from_id_internal(_func) + return dojo_model_to_id_internal(_func) # decorator with parameters needs another wrapper layer @@ -96,7 +96,7 @@ def __wrapper__(*args, **kwargs): model_or_id = get_parameter_froms_args_kwargs(args, kwargs, parameter) if model_or_id: - if not isinstance(model_or_id, models.Model) and we_want_async(): + if not isinstance(model_or_id, models.Model) and we_want_async(*args, **kwargs): logger.debug('instantiating model_or_id: %s for model: %s', model_or_id, model) try: instance = model.objects.get(id=model_or_id) diff --git a/dojo/filters.py b/dojo/filters.py index cd0958050c0..252f7f6d7a3 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -15,7 +15,7 @@ from django_filters.filters import ChoiceFilter, _truncate, DateTimeFilter import pytz from django.db.models import Q -from dojo.models import Dojo_User, Product_Type, Finding, Product, Test_Type, \ +from dojo.models import Dojo_User, Finding_Group, Product_Type, Finding, Product, Test_Type, \ Endpoint, Development_Environment, Finding_Template, Report, Note_Type, \ Engagement_Survey, Question, TextQuestion, ChoiceQuestion, Endpoint_Status, Engagement, \ ENGAGEMENT_STATUS_CHOICES, Test, App_Analysis, SEVERITY_CHOICES @@ -735,6 +735,8 @@ class ApiFindingFilter(DojoFilter): test__test_type = NumberInFilter(field_name='test__test_type', lookup_expr='in') test__engagement = NumberInFilter(field_name='test__engagement', lookup_expr='in') test__engagement__product = NumberInFilter(field_name='test__engagement__product', lookup_expr='in') + finding_group = NumberInFilter(field_name='finding_group', lookup_expr='in') + # ReportRiskAcceptanceFilter risk_acceptance = ReportRiskAcceptanceFilter() @@ -798,6 +800,17 @@ class OpenFindingFilter(DojoFilter): test__engagement = ModelMultipleChoiceFilter( queryset=Engagement.objects.none(), label="Engagement") + + if settings.FEATURE_FINDING_GROUPS: + finding_group = ModelMultipleChoiceFilter( + queryset=Finding_Group.objects.none(), + label="Finding Group") + + has_finding_group = BooleanFilter(field_name='finding_group', + lookup_expr='isnull', + exclude=True, + label='is Grouped') + risk_acceptance = ReportRiskAcceptanceFilter( label="Risk Accepted") @@ -808,6 +821,12 @@ class OpenFindingFilter(DojoFilter): exclude=True, label='has JIRA') + if settings.FEATURE_FINDING_GROUPS: + has_jira_group_issue = BooleanFilter(field_name='finding_group__jira_issue', + lookup_expr='isnull', + exclude=True, + label='has Group JIRA') + has_component = BooleanFilter(field_name='component_name', lookup_expr='isnull', exclude=True, @@ -897,9 +916,21 @@ def __init__(self, *args, **kwargs): if self.form.fields.get('test__engagement__product'): self.form.fields['test__engagement__product'].queryset = get_authorized_products(Permissions.Product_View) if self.user is not None and not self.user.is_staff: + + if self.form.fields.get('finding_group', None): + logger.debug('setting queryset for finding_group field') + self.form.fields['test__engagement__product'].queryset = \ + Finding_Group.objects.filter( + Q(test__engagement__product__authorized_users__in=[self.user]) | Q(test__engagement__product__prod_type__authorized_users__in=[self.user]) + ) + self.form.fields['endpoints'].queryset = Endpoint.objects.filter( product__authorized_users__in=[self.user]).distinct() + else: + if self.form.fields.get('finding_group', None): + self.form.fields['finding_group'].queryset = Finding_Group.objects.all() + # Don't show the product filter on the product finding view if self.pid: del self.form.fields['test__engagement__product'] diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 259a023d72f..09ac2ce5d27 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -1,9 +1,11 @@ from dojo.celery import app from dojo.decorators import dojo_async_task, dojo_model_from_id, dojo_model_to_id import logging +from time import strftime from django.utils import timezone from django.conf import settings from fieldsignals import pre_save_changed +from dojo.models import Finding, Finding_Group from dojo.utils import get_current_user from dojo.models import Finding, System_Settings @@ -110,6 +112,7 @@ def can_edit_mitigated_data(user): return settings.EDITABLE_MITIGATED_DATA and user.is_superuser + @dojo_model_to_id @dojo_async_task @app.task @@ -154,3 +157,67 @@ def post_process_finding_save(finding, dedupe_option=True, false_history=False, logger.debug('pushing finding %s to jira from finding.save()', finding.pk) import dojo.jira_link.helper as jira_helper jira_helper.push_to_jira(finding) + + +def create_finding_group(finds, finding_group_name): + logger.debug('creating finding_group_create') + if not finds or len(finds) == 0: + raise ValueError('cannot create empty Finding Group') + + finding_group_name_dummy = 'bulk group ' + strftime("%a, %d %b %Y %X", timezone.now().timetuple()) + + finding_group = Finding_Group(test=finds[0].test) + finding_group.creator = get_current_user() + finding_group.name = finding_group_name + finding_group_name_dummy + finding_group.save() + available_findings = [find for find in finds if not find.finding_group_set.all()] + finding_group.findings.set(available_findings) + + # if user provided a name, we use that, else: + # if we have components, we may set a nice name but catch 'name already exist' exceptions + try: + if finding_group_name: + finding_group.name = finding_group_name + elif finding_group.components: + finding_group.name = finding_group.components + finding_group.save() + except: + pass + + added = len(available_findings) + skipped = len(finds) - added + return finding_group, added, skipped + + +def add_to_finding_group(finding_group, finds): + added = 0 + skipped = 0 + available_findings = [find for find in finds if not find.finding_group_set.all()] + finding_group.findings.add(*available_findings) + + added = len(available_findings) + skipped = len(finds) - added + return finding_group, added, skipped + + +def remove_from_finding_group(finds): + removed = 0 + skipped = 0 + affected_groups = [] + for find in finds: + groups = find.finding_group_set.all() + if not groups: + skipped += 1 + continue + + for group in find.finding_group_set.all(): + group.findings.remove(find) + affected_groups.append(group) + + removed += 1 + + return affected_groups, removed, skipped + +# def delete_finding_group(finding_group): +# pass + diff --git a/dojo/finding/views.py b/dojo/finding/views.py index 55ca933bc91..bca16e24906 100644 --- a/dojo/finding/views.py +++ b/dojo/finding/views.py @@ -25,7 +25,7 @@ from django.views.decorators.http import require_POST from itertools import chain from dojo.user.helper import user_must_be_authorized -from dojo.utils import add_error_message_to_response, add_field_errors_to_response, close_external_issue, reopen_external_issue +from dojo.utils import add_error_message_to_response, add_field_errors_to_response, add_success_message_to_response, close_external_issue, redirect, reopen_external_issue import copy from dojo.filters import OpenFindingFilter, OpenFindingSuperFilter, AcceptedFindingFilter, AcceptedFindingSuperFilter, \ @@ -34,7 +34,7 @@ DeleteFindingTemplateForm, FindingImageFormSet, JIRAFindingForm, GITHUBFindingForm, ReviewFindingForm, ClearFindingReviewForm, \ DefectFindingForm, StubFindingForm, DeleteFindingForm, DeleteStubFindingForm, ApplyFindingTemplateForm, \ FindingFormID, FindingBulkUpdateForm, MergeFindings -from dojo.models import Finding, Notes, NoteHistory, Note_Type, \ +from dojo.models import Finding, Finding_Group, Notes, NoteHistory, Note_Type, \ BurpRawRequestResponse, Stub_Finding, Endpoint, Finding_Template, FindingImage, Endpoint_Status, \ FindingImageAccessToken, GITHUB_PKey, GITHUB_Issue, Dojo_User, Cred_Mapping, Test, Product, User, Engagement from dojo.utils import get_page_items, add_breadcrumb, FileIterWrapper, process_notifications, \ @@ -219,6 +219,7 @@ def prefetch_for_findings(findings, prefetch_type='all'): prefetched_findings = prefetched_findings.annotate(mitigated_endpoint_count=Count('endpoint_status__id', filter=Q(endpoint_status__mitigated=True))) prefetched_findings = prefetched_findings.prefetch_related('test__engagement__product__authorized_users') prefetched_findings = prefetched_findings.prefetch_related('test__engagement__product__prod_type__authorized_users') + prefetched_findings = prefetched_findings.prefetch_related('finding_group_set') else: logger.debug('unable to prefetch because query was already executed') @@ -310,8 +311,12 @@ def view_finding(request, fid): finding.last_reviewed = new_note.date finding.last_reviewed_by = user finding.save() + if finding.has_jira_issue: jira_helper.add_comment(finding, new_note) + elif finding.has_jira_group_issue: + jira_helper.add_comment(finding.finding_group, new_note) + if note_type_activation: form = TypedNoteForm(available_note_types=available_note_types) else: @@ -359,7 +364,7 @@ def view_finding(request, fid): product_tab = Product_Tab(finding.test.engagement.product.id, title="View Finding", tab="findings") - can_be_pushed_to_jira, can_be_pushed_to_jira_error, error_code = jira_helper.finding_can_be_pushed_to_jira(finding) + can_be_pushed_to_jira, can_be_pushed_to_jira_error, error_code = jira_helper.can_be_pushed_to_jira(finding) lastPos = (len(findings)) - 1 return render( @@ -527,7 +532,11 @@ def defect_finding_review(request, fid): new_note.entry = new_note.entry + "\nJira issue re-opened." # Update Dojo and Jira with a notes - jira_helper.add_comment(finding, new_note, force_push=True) + if finding.has_jira_issue: + jira_helper.add_comment(finding, new_note, force_push=True) + elif finding.has_jira_group_issue: + jira_helper.add_comment(finding.finding_group, new_note, force_push=True) + finding.save() messages.add_message( @@ -754,29 +763,32 @@ def edit_finding(request, fid): logger.debug('push_to_jira: %s', push_to_jira) logger.debug('push_all_jira_issues: %s', push_all_jira_issues) + logger.debug('has_jira_group_issue: %s', new_finding.has_jira_group_issue) # if the jira issue key was changed, update database new_jira_issue_key = jform.cleaned_data.get('jira_issue') - if new_finding.has_jira_issue: - jira_issue = new_finding.jira_issue - - # everything in DD around JIRA integration is based on the internal id of the issue in JIRA - # instead of on the public jira issue key. - # I have no idea why, but it means we have to retrieve the issue from JIRA to get the internal JIRA id. - # we can assume the issue exist, which is already checked in the validation of the jform - - if not new_jira_issue_key: - jira_helper.finding_unlink_jira(request, new_finding) - jira_message = 'Link to JIRA issue removed successfully.' - - elif new_jira_issue_key != new_finding.jira_issue.jira_key: - jira_helper.finding_unlink_jira(request, new_finding) - jira_helper.finding_link_jira(request, new_finding, new_jira_issue_key) - jira_message = 'Changed JIRA link successfully.' - else: - if new_jira_issue_key: - jira_helper.finding_link_jira(request, new_finding, new_jira_issue_key) - jira_message = 'Linked a JIRA issue successfully.' + # we only support linking / changing if there is no group issue + if not new_finding.has_jira_group_issue: + if new_finding.has_jira_issue: + jira_issue = new_finding.jira_issue + + # everything in DD around JIRA integration is based on the internal id of the issue in JIRA + # instead of on the public jira issue key. + # I have no idea why, but it means we have to retrieve the issue from JIRA to get the internal JIRA id. + # we can assume the issue exist, which is already checked in the validation of the jform + + if not new_jira_issue_key: + jira_helper.finding_unlink_jira(request, new_finding) + jira_message = 'Link to JIRA issue removed successfully.' + + elif new_jira_issue_key != new_finding.jira_issue.jira_key: + jira_helper.finding_unlink_jira(request, new_finding) + jira_helper.finding_link_jira(request, new_finding, new_jira_issue_key) + jira_message = 'Changed JIRA link successfully.' + else: + if new_jira_issue_key: + jira_helper.finding_link_jira(request, new_finding, new_jira_issue_key) + jira_message = 'Linked a JIRA issue successfully.' if 'githubform-push_to_github' in request.POST: gform = GITHUBFindingForm( @@ -787,8 +799,18 @@ def edit_finding(request, fid): else: add_external_issue(new_finding, 'github') + # if there's a finding group, that's what we need to push + push_group_to_jira = push_to_jira and new_finding.finding_group + # any existing finding should be updated + push_to_jira = push_to_jira and not push_group_to_jira and not new_finding.has_jira_issue + new_finding.save(push_to_jira=push_to_jira) + # we only push the group after storing the finding to make sure + # the updated data of the finding is pushed as part of the group + if push_group_to_jira: + jira_helper.push_to_jira(new_finding.finding_group) + messages.add_message( request, messages.SUCCESS, @@ -1800,6 +1822,8 @@ def merge_finding_product(request, pid): def finding_bulk_update_all(request, pid=None): form = FindingBulkUpdateForm(request.POST) now = timezone.now() + return_url = None + if request.method == "POST": finding_to_update = request.POST.getlist('finding_to_update') finds = Finding.objects.filter(id__in=finding_to_update).order_by("id") @@ -1886,6 +1910,51 @@ def finding_bulk_update_all(request, pid=None): for prod in prods: calculate_grade(prod) + if form.cleaned_data['finding_group_create']: + logger.debug('finding_group_create checked!') + finding_group_name = form.cleaned_data['finding_group_create_name'] + logger.debug('finding_group_create_name: %s', finding_group_name) + finding_group, added, skipped = finding_helper.create_finding_group(finds, finding_group_name) + + if added: + add_success_message_to_response('Created finding group with %s findings' % added) + return_url = reverse('view_finding_group', args=(finding_group.id,)) + + if skipped: + add_success_message_to_response('Skipped %s findings in group creation, findings already part of another group' % skipped) + + # refresh findings from db + finds = finds.all() + + if form.cleaned_data['finding_group_add']: + logger.debug('finding_group_add checked!') + fgid = form.cleaned_data['add_to_finding_group'] + finding_group = Finding_Group.objects.get(id=fgid) + finding_group, added, skipped = finding_helper.add_to_finding_group(finding_group, finds) + + if added: + add_success_message_to_response('Added %s findings to finding group %s' % (added, finding_group.name)) + return_url = reverse('view_finding_group', args=(finding_group.id,)) + + if skipped: + add_success_message_to_response('Skipped %s findings when adding to finding group %s, findings already part of another group' % (skipped, finding_group.name)) + + # refresh findings from db + finds = finds.all() + + if form.cleaned_data['finding_group_remove']: + logger.debug('finding_group_remove checked!') + finding_groups, removed, skipped = finding_helper.remove_from_finding_group(finds) + + if removed: + add_success_message_to_response('Removed %s findings from finding groups %s' % (removed, ','.join([finding_group.name for finding_group in finding_groups]))) + + if skipped: + add_success_message_to_response('Skipped %s findings when removing from any finding group, findings not part of any group' % (skipped)) + + # refresh findings from db + finds = finds.all() + if skipped_risk_accept_count > 0: messages.add_message(request, messages.WARNING, @@ -1912,6 +1981,44 @@ def finding_bulk_update_all(request, pid=None): finding.tags = tags finding.save() + if form.cleaned_data['severity'] or form.cleaned_data['status']: + prev_prod = None + for finding in finds: + # findings are ordered by product_id + if prev_prod != finding.test.engagement.product.id: + # TODO this can be inefficient as most findings usually have the same product + calculate_grade(finding.test.engagement.product) + prev_prod = finding.test.engagement.product.id + + error_counts = defaultdict(lambda: 0) + success_count = 0 + finding_groups = set([find.finding_group for find in finds if find.has_finding_group]) + logger.info('finding_groups: %s', finding_groups) + for group in finding_groups: + if form.cleaned_data.get('push_to_jira'): + can_be_pushed_to_jira, error_message, error_code = jira_helper.can_be_pushed_to_jira(group) + if not can_be_pushed_to_jira: + error_counts[error_message] += 1 + jira_helper.log_jira_alert(error_message, group) + else: + logger.debug('pushing to jira from finding.finding_bulk_update_all()') + jira_helper.push_to_jira(group) + success_count += 1 + + jira_helper.push_to_jira(group) + + for error_message, error_count in error_counts.items(): + add_error_message_to_response('%i finding groups could not be pushed to JIRA: %s' % (error_count, error_message)) + + if success_count > 0: + add_success_message_to_response('%i finding groups pushed to JIRA succesfully' % success_count) + + # refresh from db + finds = finds.all() + + error_counts = defaultdict(lambda: 0) + success_count = 0 + for finding in finds: from dojo.tools import tool_issue_updater tool_issue_updater.async_tool_issue_update(finding) @@ -1926,18 +2033,26 @@ def finding_bulk_update_all(request, pid=None): # can't use helper as when push_all_jira_issues is True, the checkbox gets disabled and is always false # push_to_jira = jira_helper.is_push_to_jira(new_finding, form.cleaned_data.get('push_to_jira')) - error_counts = defaultdict(lambda: 0) if jira_helper.is_push_all_issues(finding) or form.cleaned_data.get('push_to_jira'): - can_be_pushed_to_jira, error_message, error_code = jira_helper.finding_can_be_pushed_to_jira(finding) - if not can_be_pushed_to_jira: + + can_be_pushed_to_jira, error_message, error_code = jira_helper.can_be_pushed_to_jira(finding) + if finding.has_jira_group_issue and not finding.has_jira_issue: + error_message = 'finding already pushed as part of Finding Group' + error_counts[error_message] += 1 + jira_helper.log_jira_alert(error_message, finding) + elif not can_be_pushed_to_jira: error_counts[error_message] += 1 jira_helper.log_jira_alert(error_message, finding) else: logger.debug('pushing to jira from finding.finding_bulk_update_all()') jira_helper.push_to_jira(finding) + success_count += 1 + + for error_message, error_count in error_counts.items(): + add_error_message_to_response('%i findings could not be pushed to JIRA: %s' % (error_count, error_message)) - for error_message, error_count in error_counts.items(): - add_error_message_to_response('%i findings could not be pushed to JIRA: %s' % (error_count, error_message)) + if success_count > 0: + add_success_message_to_response('%i findings pushed to JIRA succesfully' % success_count) messages.add_message(request, messages.SUCCESS, @@ -1949,6 +2064,9 @@ def finding_bulk_update_all(request, pid=None): 'Unable to process bulk update. Required fields were not selected.', extra_tags='alert-danger') + if return_url: + redirect(request, return_url) + return redirect_to_return_url_or_else(request, None) diff --git a/dojo/finding_group/__init__.py b/dojo/finding_group/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dojo/finding_group/urls.py b/dojo/finding_group/urls.py new file mode 100644 index 00000000000..fa45ea673a6 --- /dev/null +++ b/dojo/finding_group/urls.py @@ -0,0 +1,13 @@ +from django.conf.urls import url + +from dojo.finding_group import views + +urlpatterns = [ + # finding group + url(r'^finding_group/(?P\d+)$', views.view_finding_group, name='view_finding_group'), + url(r'^finding_group/(?P\d+)/edit$', views.edit_finding_group, name='edit_finding_group'), + url(r'^finding_group/(?P\d+)/delete$', views.delete_finding_group, name='delete_finding_group'), + + url(r'^finding_group/(?P\d+)/jira/push$', views.push_to_jira, name='finding_group_push_to_jira'), + url(r'^finding_group/(?P\d+)/jira/unlink$', views.unlink_jira, name='finding_group_unlink_jira'), +] diff --git a/dojo/finding_group/views.py b/dojo/finding_group/views.py new file mode 100644 index 00000000000..2ec5bb2ae4b --- /dev/null +++ b/dojo/finding_group/views.py @@ -0,0 +1,138 @@ +# # findings +from dojo.utils import Product_Tab +from dojo.forms import DeleteFindingGroupForm +from dojo.notifications.helper import create_notification +from django.contrib import messages +from django.contrib.admin.utils import NestedObjects +from django.db.utils import DEFAULT_DB_ALIAS +from django.http.response import HttpResponse, HttpResponseRedirect, JsonResponse +from django.shortcuts import get_object_or_404, render +from django.urls.base import reverse +from django.views.decorators.http import require_POST +from dojo.models import Finding_Group +import logging +from dojo.user.helper import user_must_be_authorized +import dojo.jira_link.helper as jira_helper + +logger = logging.getLogger(__name__) + + +@user_must_be_authorized(Finding_Group, 'view', 'fgid') +def view_finding_group(request, fgid): + logger.debug('view finding group: %s', fgid) + return HttpResponse('Not implemented yet') + + +@user_must_be_authorized(Finding_Group, 'change', 'fgid') +def edit_finding_group(request, fgid): + logger.debug('edit finding group: %s', fgid) + return HttpResponse('Not implemented yet') + + +@user_must_be_authorized(Finding_Group, 'delete', 'fgid') +@require_POST +def delete_finding_group(request, fgid): + logger.debug('delete finding group: %s', fgid) + finding_group = get_object_or_404(Finding_Group, pk=fgid) + form = DeleteFindingGroupForm(instance=finding_group) + + if request.method == 'POST': + if 'id' in request.POST and str(finding_group.id) == request.POST['id']: + form = DeleteFindingGroupForm(request.POST, instance=finding_group) + if form.is_valid(): + finding_group.delete() + messages.add_message(request, + messages.SUCCESS, + 'Finding Group and relationships removed.', + extra_tags='alert-success') + + create_notification(event='other', + title='Deletion of %s' % finding_group.name, + description='The finding group "%s" was deleted by %s' % (finding_group.name, request.user), + url=request.build_absolute_uri(reverse('view_test', args=(finding_group.test.id,))), + icon="exclamation-triangle") + return HttpResponseRedirect(reverse('view_test', args=(finding_group.test.id,))) + + collector = NestedObjects(using=DEFAULT_DB_ALIAS) + collector.collect([finding_group]) + rels = collector.nested() + product_tab = Product_Tab(finding_group.test.engagement.product.id, title="Product", tab="settings") + + return render(request, 'dojo/delete_finding_group.html', + {'finding_group': finding_group, + 'form': form, + 'product_tab': product_tab, + 'rels': rels, + }) + + +@user_must_be_authorized(Finding_Group, 'change', 'fgid') +@require_POST +def unlink_jira(request, fgid): + logger.debug('/finding_group/%s/jira/unlink', fgid) + group = get_object_or_404(Finding_Group, id=fgid) + logger.info('trying to unlink a linked jira issue from %d:%s', group.id, group.name) + if group.has_jira_issue: + try: + jira_helper.unlink_jira(request, group) + + messages.add_message( + request, + messages.SUCCESS, + 'Link to JIRA issue succesfully deleted', + extra_tags='alert-success') + + return JsonResponse({'result': 'OK'}) + except Exception as e: + logger.exception(e) + messages.add_message( + request, + messages.ERROR, + 'Link to JIRA could not be deleted, see alerts for details', + extra_tags='alert-danger') + + return HttpResponse(status=500) + else: + messages.add_message( + request, + messages.ERROR, + 'Link to JIRA not found', + extra_tags='alert-danger') + return HttpResponse(status=400) + + +@user_must_be_authorized(Finding_Group, 'change', 'fgid') +@require_POST +def push_to_jira(request, fgid): + logger.debug('/finding_group/%s/jira/push', fgid) + group = get_object_or_404(Finding_Group, id=fgid) + try: + logger.info('trying to push %d:%s to JIRA to create or update JIRA issue', group.id, group.name) + logger.debug('pushing to jira from group.push_to-jira()') + + # it may look like succes here, but the push_to_jira are swallowing exceptions + # but cant't change too much now without having a test suite, so leave as is for now with the addition warning message to check alerts for background errors. + if jira_helper.push_to_jira(group, sync=True): + messages.add_message( + request, + messages.SUCCESS, + message='Action queued to create or update linked JIRA issue, check alerts for background errors.', + extra_tags='alert-success') + else: + messages.add_message( + request, + messages.SUCCESS, + 'Push to JIRA failed, check alerts on the top right for errors', + extra_tags='alert-danger') + + return JsonResponse({'result': 'OK'}) + except Exception as e: + logger.exception(e) + logger.error('Error pushing to JIRA: ', exc_info=True) + messages.add_message( + request, + messages.ERROR, + 'Error pushing to JIRA', + extra_tags='alert-danger') + return HttpResponse(status=500) + # return redirect_to_return_url_or_else(request, reverse('view_finding', args=(group.id,))) diff --git a/dojo/fixtures/defect_dojo_sample_data.json b/dojo/fixtures/defect_dojo_sample_data.json index 794d9857c64..37f20ac577e 100644 --- a/dojo/fixtures/defect_dojo_sample_data.json +++ b/dojo/fixtures/defect_dojo_sample_data.json @@ -24211,7 +24211,6 @@ "username": "user3", "password": "user3", "default_issue_type": "Spike", - "issue_template": "issue-trackers/jira-description.tpl", "epic_name_id": 333, "open_status_key": 333, "close_status_key": 334, diff --git a/dojo/fixtures/dojo_testdata.json b/dojo/fixtures/dojo_testdata.json index 5367b3876a7..6cc59852960 100644 --- a/dojo/fixtures/dojo_testdata.json +++ b/dojo/fixtures/dojo_testdata.json @@ -1567,7 +1567,6 @@ "username": "", "password": "", "default_issue_type": "Task", - "issue_template": "issue-trackers/jira-description.tpl", "epic_name_id": 10011, "open_status_key": 11, "close_status_key": 41, @@ -1585,7 +1584,6 @@ "fields": { "username": "defect.dojo", "default_issue_type": "Task", - "issue_template": "issue-trackers/jira-description.tpl", "finding_text": "", "low_mapping_severity": "test severity", "url": "http://www.testjira.com", @@ -1604,7 +1602,6 @@ "fields": { "username": "defect.dojo", "default_issue_type": "Spike", - "issue_template": "issue-trackers/jira-description.tpl", "finding_text": "", "low_mapping_severity": "test severity", "url": "http://www.testjira.com", diff --git a/dojo/forms.py b/dojo/forms.py index 7ebd5d0d399..9c9ac270b11 100755 --- a/dojo/forms.py +++ b/dojo/forms.py @@ -21,7 +21,7 @@ from django.utils import timezone import tagulous -from dojo.models import Finding, Product_Type, Product, Note_Type, \ +from dojo.models import Finding, Finding_Group, Product_Type, Product, Note_Type, \ Check_List, User, Engagement, Test, Test_Type, Notes, Risk_Acceptance, \ Development_Environment, Dojo_User, Endpoint, Stub_Finding, Finding_Template, Report, FindingImage, \ JIRA_Issue, JIRA_Project, JIRA_Instance, GITHUB_Issue, GITHUB_PKey, GITHUB_Conf, UserContactInfo, Tool_Type, \ @@ -295,6 +295,15 @@ class Meta: 'enable_simple_risk_acceptance', 'enable_full_risk_acceptance'] +class DeleteFindingGroupForm(forms.ModelForm): + id = forms.IntegerField(required=True, + widget=forms.widgets.HiddenInput()) + + class Meta: + model = Finding_Group + fields = ['id'] + + class Edit_Product_MemberForm(forms.ModelForm): user = forms.ModelChoiceField(queryset=None, required=True) role = forms.ChoiceField(choices=Roles.choices()) @@ -1176,6 +1185,13 @@ class FindingBulkUpdateForm(forms.ModelForm): risk_accept = forms.BooleanField(required=False) risk_unaccept = forms.BooleanField(required=False) + finding_group = forms.BooleanField(required=False) + finding_group_create = forms.BooleanField(required=False) + finding_group_create_name = forms.CharField(required=False) + finding_group_add = forms.BooleanField(required=False) + add_to_finding_group = forms.BooleanField(required=False) + finding_group_remove = forms.BooleanField(required=False) + push_to_jira = forms.BooleanField(required=False) # unlink_from_jira = forms.BooleanField(required=False) push_to_github = forms.BooleanField(required=False) @@ -1740,19 +1756,25 @@ class Meta: 'high_mapping_severity', 'critical_mapping_severity', 'finding_text'] -def get_jira_issue_template_choices(): - template_dir = settings.JIRA_TEMPLATE_DIR - template_list = [('', '---')] - for base_dir, dirnames, filenames in os.walk(template_dir): - for filename in filenames: +def get_jira_issue_template_dir_choices(): + template_root = settings.JIRA_TEMPLATE_ROOT + template_dir_list = [('', '---')] + for base_dir, dirnames, filenames in os.walk(template_root): + # for filename in filenames: + # if base_dir.startswith(settings.TEMPLATE_DIR_PREFIX): + # base_dir = base_dir[len(settings.TEMPLATE_DIR_PREFIX):] + # template_list.append((os.path.join(base_dir, filename), filename)) + + for dirname in dirnames: if base_dir.startswith(settings.TEMPLATE_DIR_PREFIX): base_dir = base_dir[len(settings.TEMPLATE_DIR_PREFIX):] - template_list.append((os.path.join(base_dir, filename), filename)) - logger.debug('templates: %s', template_list) - return template_list + template_dir_list.append((os.path.join(base_dir, dirname), dirname)) + logger.debug('templates: %s', template_dir_list) + return template_dir_list -JIRA_TEMPLATE_CHOICES = sorted(get_jira_issue_template_choices()) + +JIRA_TEMPLATE_CHOICES = sorted(get_jira_issue_template_dir_choices()) class JIRA_IssueForm(forms.ModelForm): @@ -1763,9 +1785,9 @@ class Meta: class JIRAForm(forms.ModelForm): - issue_template = forms.ChoiceField(required=False, + issue_template_dir = forms.ChoiceField(required=False, choices=JIRA_TEMPLATE_CHOICES, - help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.') + help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.') password = forms.CharField(widget=forms.PasswordInput, required=True) @@ -2122,9 +2144,9 @@ class Meta: class JIRAProjectForm(forms.ModelForm): jira_instance = forms.ModelChoiceField(queryset=JIRA_Instance.objects.all(), label='JIRA Instance', required=False) - issue_template = forms.ChoiceField(required=False, + issue_template_dir = forms.ChoiceField(required=False, choices=JIRA_TEMPLATE_CHOICES, - help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.') + help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.') prefix = 'jira-project-form' @@ -2213,6 +2235,12 @@ def __init__(self, *args, **kwargs): self.fields['jira_issue'].widget = forms.TextInput(attrs={'placeholder': 'Leave empty and check push to jira to create a new JIRA issue'}) + if self.instance.has_jira_group_issue: + self.fields['push_to_jira'].widget.attrs['checked'] = 'checked' + self.fields['jira_issue'].help_text = 'Changing the linked JIRA issue for finding groups is not (yet) supported.' + self.initial['jira_issue'] = self.instance.finding_group.jira_issue.jira_key + self.fields['jira_issue'].disabled = True + def clean(self): logger.debug('jform clean') import dojo.jira_link.helper as jira_helper @@ -2223,14 +2251,22 @@ def clean(self): logger.debug('self.cleaned_data.push_to_jira: %s', self.cleaned_data.get('push_to_jira', None)) - if self.cleaned_data.get('push_to_jira', None): - can_be_pushed_to_jira, error_message, error_code = jira_helper.finding_can_be_pushed_to_jira(self.instance, self.finding_form) + if self.cleaned_data.get('push_to_jira', None) and finding.has_jira_group_issue: + can_be_pushed_to_jira, error_message, error_code = jira_helper.can_be_pushed_to_jira(self.instance.finding_group, self.finding_form) + if not can_be_pushed_to_jira: + self.add_error('push_to_jira', ValidationError(error_message, code=error_code)) + # for field in error_fields: + # self.finding_form.add_error(field, error) + + elif self.cleaned_data.get('push_to_jira', None): + can_be_pushed_to_jira, error_message, error_code = jira_helper.can_be_pushed_to_jira(self.instance, self.finding_form) if not can_be_pushed_to_jira: self.add_error('push_to_jira', ValidationError(error_message, code=error_code)) # for field in error_fields: # self.finding_form.add_error(field, error) - if jira_issue_key_new: + if jira_issue_key_new and not finding.has_jira_group_issue: + # when there is a group jira issue, we skip all the linking/unlinking as this is not supported (yet) if finding: # in theory there can multiple jira instances that have similar projects # so checking by only the jira issue key can lead to false positives diff --git a/dojo/jira_link/helper.py b/dojo/jira_link/helper.py index 92bb33c4786..a8b01be5674 100644 --- a/dojo/jira_link/helper.py +++ b/dojo/jira_link/helper.py @@ -1,6 +1,6 @@ import logging -from dojo.utils import add_error_message_to_response, get_system_setting +from dojo.utils import add_error_message_to_response, get_system_setting, to_str_typed import os import io import json @@ -10,14 +10,14 @@ from django.utils import timezone from jira import JIRA from jira.exceptions import JIRAError -from dojo.models import Finding, Risk_Acceptance, Stub_Finding, Test, Engagement, Product, JIRA_Issue, JIRA_Project, \ - System_Settings, Notes, JIRA_Instance, User +from dojo.models import Finding, Finding_Group, Risk_Acceptance, Stub_Finding, Test, Engagement, Product, \ + JIRA_Issue, JIRA_Project, System_Settings, Notes, JIRA_Instance, User from requests.auth import HTTPBasicAuth from dojo.notifications.helper import create_notification from django.contrib import messages from dojo.celery import app from dojo.decorators import dojo_async_task, dojo_model_from_id, dojo_model_to_id -from dojo.utils import truncate_with_dots +from dojo.utils import truncate_with_dots, prod_name from django.urls import reverse from dojo.forms import JIRAProjectForm, JIRAEngagementForm @@ -83,40 +83,52 @@ def is_push_all_issues(instance): # optionally provides a form with the new data for the finding # any finding that already has a JIRA issue can be pushed again to JIRA # returns True/False, error_message, error_code -def finding_can_be_pushed_to_jira(finding, form=None): +def can_be_pushed_to_jira(obj, form=None): # logger.debug('can be pushed to JIRA: %s', finding_or_form) - if not get_jira_project(finding): - return False, 'Finding cannot be pushed to jira as there is no jira project configuration for this product.', 'error_no_jira_project' + if not get_jira_project(obj): + return False, '%s cannot be pushed to jira as there is no jira project configuration for this product.' % to_str_typed(obj), 'error_no_jira_project' - if isinstance(finding, Stub_Finding): + if not hasattr(obj, 'has_jira_issue'): + return False, '%s cannot be pushed to jira as there is no jira_issue attribute.' % to_str_typed(obj), 'error_no_jira_issue_attribute' + + if isinstance(obj, Stub_Finding): # stub findings don't have active/verified/etc and can always be pushed return True, None, None - if finding.has_jira_issue: + if obj.has_jira_issue: return True, None, None - if form: - active = form['active'].value() - verified = form['verified'].value() - severity = form['severity'].value() - else: - active = finding.active - verified = finding.verified - severity = finding.severity + if type(obj) == Finding: + if form: + active = form['active'].value() + verified = form['verified'].value() + severity = form['severity'].value() + else: + active = obj.active + verified = obj.verified + severity = obj.severity - logger.debug('finding_can_be_pushed_to_jira: %s, %s, %s', active, verified, severity) + logger.debug('can_be_pushed_to_jira: %s, %s, %s', active, verified, severity) - if not active or not verified: - logger.debug('Findings must be active and verified to be pushed to JIRA') - return False, 'Findings must be active and verified to be pushed to JIRA', 'not_active_or_verified' + if not active or not verified: + logger.debug('Findings must be active and verified to be pushed to JIRA') + return False, 'Findings must be active and verified to be pushed to JIRA', 'not_active_or_verified' - jira_minimum_threshold = None - if System_Settings.objects.get().jira_minimum_severity: - jira_minimum_threshold = Finding.get_number_severity(System_Settings.objects.get().jira_minimum_severity) + jira_minimum_threshold = None + if System_Settings.objects.get().jira_minimum_severity: + jira_minimum_threshold = Finding.get_number_severity(System_Settings.objects.get().jira_minimum_severity) - if jira_minimum_threshold and jira_minimum_threshold > Finding.get_number_severity(severity): - logger.debug('Finding below the minimum JIRA severity threshold (%s).' % System_Settings.objects.get().jira_minimum_severity) - return False, 'Finding below the minimum JIRA severity threshold (%s).' % System_Settings.objects.get().jira_minimum_severity, 'below_minimum_threshold' + if jira_minimum_threshold and jira_minimum_threshold > Finding.get_number_severity(severity): + logger.debug('Finding below the minimum JIRA severity threshold (%s).' % System_Settings.objects.get().jira_minimum_severity) + return False, 'Finding below the minimum JIRA severity threshold (%s).' % System_Settings.objects.get().jira_minimum_severity, 'below_minimum_threshold' + elif type(obj) == Finding_Group: + if not obj.findings.all(): + return False, '%s cannot be pushed to jira as it is empty.' % to_str_typed(obj), 'error_empty' + if 'Active' not in obj.status(): + return False, '%s cannot be pushed to jira as it is not active.' % to_str_typed(obj), 'error_inactive' + + else: + return False, '%s cannot be pushed to jira as it is of unsupported type.' % to_str_typed(obj), 'error_unsupported' return True, None, None @@ -139,6 +151,9 @@ def get_jira_project(obj, use_inheritance=True): finding = obj return get_jira_project(finding.test) + if isinstance(obj, Finding_Group): + return get_jira_project(obj.test) + if isinstance(obj, Test): test = obj return get_jira_project(test.engagement) @@ -260,27 +275,31 @@ def get_jira_project_key(obj): def get_jira_issue_template(obj): jira_project = get_jira_project(obj) - template = jira_project.issue_template - if not template: + + template_dir = jira_project.issue_template_dir + if not template_dir: jira_instance = get_jira_instance(obj) - template = jira_instance.issue_template + template_dir = jira_instance.issue_template_dir # fallback to default as before - if not template: - return 'issue-trackers/jira-description.tpl' + if not template_dir: + template_dir = 'issue-trackers/jira_full/' - return template + if isinstance(obj, Finding_Group): + return os.path.join(template_dir, 'jira-finding-group-description.tpl') + else: + return os.path.join(template_dir, 'jira-description.tpl') def get_jira_creation(obj): - if isinstance(obj, Finding) or isinstance(obj, Engagement): + if isinstance(obj, Finding) or isinstance(obj, Engagement) or isinstance(obj, Finding_Group): if obj.has_jira_issue: return obj.jira_issue.jira_creation return None def get_jira_change(obj): - if isinstance(obj, Finding) or isinstance(obj, Engagement): + if isinstance(obj, Finding) or isinstance(obj, Engagement) or isinstance(obj, Finding_Group): if obj.has_jira_issue: return obj.jira_issue.jira_change else: @@ -300,7 +319,7 @@ def has_jira_issue(obj): def get_jira_issue(obj): - if isinstance(obj, Finding) or isinstance(obj, Engagement): + if isinstance(obj, Finding) or isinstance(obj, Engagement) or isinstance(obj, Finding_Group): try: return obj.jira_issue except JIRA_Issue.DoesNotExist: @@ -405,17 +424,15 @@ def log_jira_generic_alert(title, description): # Logs the error to the alerts table, which appears in the notification toolbar -def log_jira_alert(error, finding): - prod_name = finding.test.engagement.product.name if finding else 'unknown' - +def log_jira_alert(error, obj): create_notification( event='jira_update', - title='Error pushing to JIRA ' + '(' + truncate_with_dots(prod_name, 25) + ')', - description='Finding: ' + str(finding.id if finding else 'unknown') + ', ' + error, - url=reverse('view_finding', args=(finding.id, )) if finding else None, + title='Error pushing to JIRA ' + '(' + truncate_with_dots(prod_name(obj), 25) + ')', + description=to_str_typed(obj) + ', ' + error, + url=obj.get_absolute_url, icon='bullseye', source='Push to JIRA', - finding=finding) + obj=obj) # Displays an alert for Jira notifications @@ -429,7 +446,7 @@ def log_jira_message(text, finding): source='JIRA', finding=finding) -def get_labels(find): +def get_labels(obj): # Update Label with system setttings label labels = [] system_settings = System_Settings.objects.get() @@ -442,67 +459,125 @@ def get_labels(find): for system_label in system_labels: labels.append(system_label) # Update the label with the product name (underscore) - prod_name = find.test.engagement.product.name.replace(" ", "_") - labels.append(prod_name) + labels.append(prod_name(obj).replace(" ", "_")) return labels -def jira_description(find): - template = get_jira_issue_template(find) +def jira_summary(obj): + if type(obj) == Finding: + return obj.title + + if type(obj) == Finding_Group: + return obj.name + + return None + + +def jira_description(obj): + template = get_jira_issue_template(obj) logger.debug('rendering description for jira from: %s', template) kwargs = {} - kwargs['finding'] = find + if isinstance(obj, Finding): + kwargs['finding'] = obj + elif isinstance(obj, Finding_Group): + kwargs['finding_group'] = obj + description = render_to_string(template, kwargs) logger.debug('rendered description: %s', description) return description -def push_to_jira(obj): +def jira_priority(obj): + return get_jira_instance(obj).get_priority(obj.severity) + + +def jira_environment(obj): + if type(obj) == Finding: + return "\n".join([str(endpoint) for endpoint in obj.endpoints.all()]) + elif type(obj) == Finding_Group: + return "\n".join([jira_environment(finding) for finding in obj.findings.all()]) + else: + return '' + + +def push_to_jira(obj, *args, **kwargs): if isinstance(obj, Finding): finding = obj if finding.has_jira_issue: - return update_jira_issue(finding) + return update_jira_issue_for_finding(finding, *args, **kwargs) else: - return add_jira_issue(finding) + return add_jira_issue_for_finding(finding, *args, **kwargs) elif isinstance(obj, Engagement): engagement = obj if engagement.has_jira_issue: - return update_epic(engagement) + return update_epic(engagement, *args, **kwargs) + else: + return add_epic(engagement, *args, **kwargs) + + elif isinstance(obj, Finding_Group): + group = obj + if group.has_jira_issue: + return update_jira_issue_for_finding_group(group, *args, **kwargs) else: - return add_epic(engagement) + return add_jira_issue_for_finding_group(group, *args, **kwargs) else: logger.error('unsupported object passed to push_to_jira: %s %i %s', obj.__name__, obj.id, obj) +def add_issues_to_epic(jira, obj, epic_id, issue_keys, ignore_epics=True): + try: + return jira.add_issues_to_epic(epic_id=epic_id, issue_keys=issue_keys, ignore_epics=ignore_epics) + except JIRAError as e: + logger.error('error adding issues %s to epic %s for %s', issue_keys, epic_id, obj.id) + logger.exception(e) + log_jira_alert(e.text, obj) + return False + + +# we need two separate celery tasks due to the decorators we're using to map to/from ids + @dojo_model_to_id @dojo_async_task @app.task @dojo_model_from_id -def add_jira_issue(find): - logger.info('trying to create a new jira issue for %d:%s', find.id, find.title) +def add_jira_issue_for_finding(finding, *args, **kwargs): + return add_jira_issue(finding, *args, **kwargs) + + +@dojo_model_to_id +@dojo_async_task +@app.task +@dojo_model_from_id(model=Finding_Group) +def add_jira_issue_for_finding_group(finding_group, *args, **kwargs): + return add_jira_issue(finding_group, *args, **kwargs) + + +def add_jira_issue(obj, *args, **kwargs): + logger.info('trying to create a new jira issue for %d:%s', obj.id, to_str_typed(obj)) if not is_jira_enabled(): return False - if not is_jira_configured_and_enabled(find): - logger.error("Finding {} cannot be pushed to JIRA as there is no JIRA configuration for this product.".format(find.id)) - log_jira_alert('Finding cannot be pushed to JIRA as there is no JIRA configuration for this product.', find) + if not is_jira_configured_and_enabled(obj): + message = 'Object %s cannot be pushed to JIRA as there is no JIRA configuration for %s.' % (obj.id, to_str_typed(obj)) + logger.error(message) + log_jira_alert(message, obj) return False - jira_project = get_jira_project(find) - jira_instance = get_jira_instance(find) + jira_project = get_jira_project(obj) + jira_instance = get_jira_instance(obj) - can_be_pushed_to_jira, error_message, error_code = finding_can_be_pushed_to_jira(find) - if not can_be_pushed_to_jira: - log_jira_alert(error_message, find) - logger.warn("Finding %s cannot be pushed to JIRA: %s.", find.id, error_message) + obj_can_be_pushed_to_jira, error_message, error_code = can_be_pushed_to_jira(obj) + if not obj_can_be_pushed_to_jira: + log_jira_alert(error_message, obj) + logger.warn("%s cannot be pushed to JIRA: %s.", to_str_typed(obj), error_message) logger.warn("The JIRA issue will NOT be created.") return False - logger.debug('Trying to create a new JIRA issue for finding {}...'.format(find.id)) + logger.debug('Trying to create a new JIRA issue for %s...', to_str_typed(obj)) meta = None try: JIRAError.log_to_tempfile = False @@ -512,8 +587,8 @@ def add_jira_issue(find): 'project': { 'key': jira_project.project_key }, - 'summary': find.title, - 'description': jira_description(find), + 'summary': jira_summary(obj), + 'description': jira_description(obj), 'issuetype': { 'name': jira_instance.default_issue_type }, @@ -538,10 +613,10 @@ def add_jira_issue(find): if 'priority' in meta['projects'][0]['issuetypes'][0]['fields']: fields['priority'] = { - 'name': jira_instance.get_priority(find.severity) + 'name': jira_priority(obj) } - labels = get_labels(find) + labels = get_labels(obj) if labels: if 'labels' in meta['projects'][0]['issuetypes'][0]['fields']: fields['labels'] = labels @@ -550,72 +625,94 @@ def add_jira_issue(find): if 'duedate' in meta['projects'][0]['issuetypes'][0]['fields']: # jira wants YYYY-MM-DD - duedate = find.sla_deadline() + duedate = obj.sla_deadline() if duedate: fields['duedate'] = duedate.strftime('%Y-%m-%d') - if len(find.endpoints.all()) > 0: - if not meta: - meta = get_jira_meta(jira, jira_project) + if not meta: + meta = get_jira_meta(jira, jira_project) - if 'environment' in meta['projects'][0]['issuetypes'][0]['fields']: - environment = "\n".join([str(endpoint) for endpoint in find.endpoints.all()]) - fields['environment'] = environment + if 'environment' in meta['projects'][0]['issuetypes'][0]['fields']: + fields['environment'] = jira_environment(obj) logger.debug('sending fields to JIRA: %s', fields) new_issue = jira.create_issue(fields) - j_issue = JIRA_Issue( - jira_id=new_issue.id, jira_key=new_issue.key, finding=find, jira_project=jira_project) - j_issue.jira_creation = timezone.now() - j_issue.jira_change = timezone.now() - j_issue.save() - issue = jira.issue(new_issue.id) - # Upload dojo finding screenshots to Jira - for pic in find.images.all(): - jira_attachment( - find, jira, issue, - settings.MEDIA_ROOT + pic.image_large.name) + findings = [obj] + if type(obj) == Finding_Group: + findings = obj.findings.all() + + for find in findings: + for pic in find.images.all(): + jira_attachment( + find, jira, new_issue, + settings.MEDIA_ROOT + pic.image_large.name) if jira_project.enable_engagement_epic_mapping: - eng = find.test.engagement + eng = obj.test.engagement logger.debug('Adding to EPIC Map: %s', eng.name) epic = get_jira_issue(eng) if epic: - jira.add_issues_to_epic(epic_id=epic.jira_id, issue_keys=[str(j_issue.jira_id)], ignore_epics=True) + add_issues_to_epic(jira, obj, epic_id=epic.jira_id, issue_keys=[str(new_issue.id)], ignore_epics=True) else: logger.info('The following EPIC does not exist: %s', eng.name) - logger.info('Created the following jira issue for %d:%s', find.id, find.title) + # only link the new issue if it was succefully created, incl attachments and epic link + logger.debug('saving JIRA_Issue for %s finding %s', new_issue.key, obj.id) + j_issue = JIRA_Issue( + jira_id=new_issue.id, jira_key=new_issue.key, jira_project=jira_project) + j_issue.set_obj(obj) + + j_issue.jira_creation = timezone.now() + j_issue.jira_change = timezone.now() + j_issue.save() + issue = jira.issue(new_issue.id) + + logger.info('Created the following jira issue for %d:%s', obj.id, to_str_typed(obj)) return True except JIRAError as e: logger.exception(e) logger.error("jira_meta for project: %s and url: %s meta: %s", jira_project.project_key, jira_project.jira_instance.url, json.dumps(meta, indent=4)) # this is None safe - log_jira_alert(e.text, find) + log_jira_alert(e.text, obj) return False +# we need two separate celery tasks due to the decorators we're using to map to/from ids + @dojo_model_to_id @dojo_async_task @app.task @dojo_model_from_id -def update_jira_issue(find): - logger.debug('trying to update a linked jira issue for %d:%s', find.id, find.title) +def update_jira_issue_for_finding(finding, *args, **kwargs): + return update_jira_issue(finding, *args, **kwargs) + + +@dojo_model_to_id +@dojo_async_task +@app.task +@dojo_model_from_id(model=Finding_Group) +def update_jira_issue_for_finding_group(finding_group, *args, **kwargs): + return update_jira_issue(finding_group, *args, **kwargs) + + +def update_jira_issue(obj, *args, **kwargs): + logger.debug('trying to update a linked jira issue for %d:%s', obj.id, to_str_typed(obj)) if not is_jira_enabled(): return False - jira_project = get_jira_project(find) - jira_instance = get_jira_instance(find) + jira_project = get_jira_project(obj) + jira_instance = get_jira_instance(obj) - if not jira_project: - logger.error("Finding {} cannot be pushed to JIRA as there is no JIRA configuration for this product.".format(find.id)) - log_jira_alert('Finding cannot be pushed to JIRA as there is no JIRA configuration for this product.', find) + if not is_jira_configured_and_enabled(obj): + message = 'Object %s cannot be pushed to JIRA as there is no JIRA configuration for %s.' % (obj.id, to_str_typed(obj)) + logger.error(message) + log_jira_alert(message, obj) return False - j_issue = find.jira_issue + j_issue = obj.jira_issue meta = None try: JIRAError.log_to_tempfile = False @@ -628,7 +725,7 @@ def update_jira_issue(find): if issue.fields.components: log_jira_alert( "Component not updated, exists in Jira already. Update from Jira instead.", - find) + obj) elif jira_project.component: # Add component to the Jira issue component = [ @@ -641,50 +738,54 @@ def update_jira_issue(find): if not meta: meta = get_jira_meta(jira, jira_project) - labels = get_labels(find) + labels = get_labels(obj) if labels: if 'labels' in meta['projects'][0]['issuetypes'][0]['fields']: fields['labels'] = labels - if len(find.endpoints.all()) > 0: - if 'environment' in meta['projects'][0]['issuetypes'][0]['fields']: - environment = "\n".join([str(endpoint) for endpoint in find.endpoints.all()]) - fields['environment'] = environment - - # Upload dojo finding screenshots to Jira - for pic in find.images.all(): - jira_attachment(find, jira, issue, - settings.MEDIA_ROOT + pic.image_large.name) + if 'environment' in meta['projects'][0]['issuetypes'][0]['fields']: + fields['environment'] = jira_environment(obj) logger.debug('sending fields to JIRA: %s', fields) issue.update( - summary=find.title, - description=jira_description(find), - priority={'name': jira_instance.get_priority(find.severity)}, + summary=jira_summary(obj), + description=jira_description(obj), + priority={'name': jira_priority(obj)}, fields=fields) - push_status_to_jira(find, jira_instance, jira, issue) + push_status_to_jira(obj, jira_instance, jira, issue) - find.jira_issue.jira_change = timezone.now() - find.jira_issue.save() + # Upload dojo finding screenshots to Jira + findings = [obj] + if type(obj) == Finding_Group: + findings = obj.findings.all() + + for find in findings: + for pic in find.images.all(): + jira_attachment( + find, jira, issue, + settings.MEDIA_ROOT + pic.image_large.name) if jira_project.enable_engagement_epic_mapping: eng = find.test.engagement logger.debug('Adding to EPIC Map: %s', eng.name) epic = get_jira_issue(eng) if epic: - jira.add_issues_to_epic(epic_id=epic.jira_id, issue_keys=[str(j_issue.jira_id)], ignore_epics=True) + add_issues_to_epic(jira, obj, epic_id=epic.jira_id, issue_keys=[str(j_issue.jira_id)], ignore_epics=True) else: logger.info('The following EPIC does not exist: %s', eng.name) + j_issue.jira_change = timezone.now() + j_issue.save() + logger.debug('Updated the following linked jira issue for %d:%s', find.id, find.title) return True except JIRAError as e: logger.exception(e) logger.error("jira_meta for project: %s and url: %s meta: %s", jira_project.project_key, jira_project.jira_instance.url, json.dumps(meta, indent=4)) # this is None safe - log_jira_alert(e.text, find) + log_jira_alert(e.text, obj) return False @@ -748,9 +849,8 @@ def issue_from_jira_is_active(issue_from_jira): return False -def push_status_to_jira(find, jira_instance, jira, issue, save=False): - - status_list = find.status() +def push_status_to_jira(obj, jira_instance, jira, issue, save=False): + status_list = obj.status() issue_closed = False # check RESOLVED_STATUS first to avoid corner cases with findings that are Inactive, but verified if any(item in status_list for item in RESOLVED_STATUS): @@ -771,8 +871,8 @@ def push_status_to_jira(find, jira_instance, jira, issue, save=False): updated = False if updated and save: - find.jira_issue.jira_change = timezone.now() - find.jira_issue.save() + obj.jira_issue.jira_change = timezone.now() + obj.jira_issue.save() # gets the metadata for the default issue type in this jira project @@ -1021,19 +1121,19 @@ def jira_get_issue(jira_project, issue_key): @app.task @dojo_model_from_id(model=Notes, parameter=1) @dojo_model_from_id -def add_comment(find, note, force_push=False): - if not is_jira_configured_and_enabled(find): +def add_comment(obj, note, force_push=False): + if not is_jira_configured_and_enabled(obj): return False - logger.debug('trying to add a comment to a linked jira issue for: %d:%s', find.id, find.title) + logger.debug('trying to add a comment to a linked jira issue for: %d:%s', obj.id, obj) if not note.private: - jira_project = get_jira_project(find) - jira_instance = get_jira_instance(find) + jira_project = get_jira_project(obj) + jira_instance = get_jira_instance(obj) if jira_project.push_notes or force_push is True: try: jira = get_jira_connection(jira_instance) - j_issue = find.jira_issue + j_issue = obj.jira_issue jira.add_comment( j_issue.jira_id, '(%s): %s' % (note.author.get_full_name() if note.author.get_full_name() else note.author.username, note.entry)) @@ -1089,12 +1189,14 @@ def finding_link_jira(request, finding, new_jira_issue_key): def finding_unlink_jira(request, finding): - logger.debug('removing linked jira issue %s for finding %i', finding.jira_issue.jira_key, finding.id) - finding.jira_issue.delete() - finding.save(push_to_jira=False, dedupe_option=False, issue_updater_option=False) + return unlink_jira(request, finding) - jira_issue_url = get_jira_url(finding) +def unlink_jira(request, obj): + logger.debug('removing linked jira issue %s for %i:%s', obj.jira_issue.jira_key, obj.id, to_str_typed(obj)) + obj.jira_issue.delete() + # finding.save(push_to_jira=False, dedupe_option=False, issue_updater_option=False) + # jira_issue_url = get_jira_url(finding) return True diff --git a/dojo/jira_link/views.py b/dojo/jira_link/views.py index cb476efdf42..20b7f55928d 100644 --- a/dojo/jira_link/views.py +++ b/dojo/jira_link/views.py @@ -58,33 +58,14 @@ def webhook(request, secret=None): # xml examples at the end of file jid = parsed['issue']['id'] jissue = get_object_or_404(JIRA_Issue, jira_id=jid) - logging.info("Received issue update for {}".format(jissue.jira_key)) - if jissue.finding: - finding = jissue.finding - - assignee = parsed['issue']['fields'].get('assignee') - assignee_name = assignee['name'] if assignee else None - - resolution = parsed['issue']['fields']['resolution'] - - # "resolution":{ - # "self":"http://www.testjira.com/rest/api/2/resolution/11", - # "id":"11", - # "description":"Cancelled by the customer.", - # "name":"Cancelled" - # }, - # or - # "resolution": null - - # or - # "resolution": "None" - - resolution = resolution if resolution and resolution != "None" else None - resolution_id = resolution['id'] if resolution else None - resolution_name = resolution['name'] if resolution else None - jira_now = parse_datetime(parsed['issue']['fields']['updated']) - jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now) + findings = None + if jissue.finding: + logging.info("Received issue update for {} for finding {}".format(jissue.jira_key, jissue.finding.id)) + findings = [jissue.finding] + elif jissue.finding_group: + logging.info("Received issue update for {} for finding group {}".format(jissue.jira_key, jissue.finding_group)) + findings = jissue.finding_group.findings.all() elif jissue.engagement: # if parsed['issue']['fields']['resolution'] != None: # eng.active = False @@ -92,7 +73,35 @@ def webhook(request, secret=None): # eng.save() return HttpResponse('Update for engagement ignored') else: - raise Http404('No finding or engagement found for JIRA issue {}'.format(jissue.jira_key)) + logging.info("Received issue update for {} for unknown object".format(jissue.jira_key)) + raise Http404('No finding, finding_group or engagement found for JIRA issue {}'.format(jissue.jira_key)) + + assignee = parsed['issue']['fields'].get('assignee') + assignee_name = assignee['name'] if assignee else None + + resolution = parsed['issue']['fields']['resolution'] + + # "resolution":{ + # "self":"http://www.testjira.com/rest/api/2/resolution/11", + # "id":"11", + # "description":"Cancelled by the customer.", + # "name":"Cancelled" + # }, + + # or + # "resolution": null + + # or + # "resolution": "None" + + resolution = resolution if resolution and resolution != "None" else None + resolution_id = resolution['id'] if resolution else None + resolution_name = resolution['name'] if resolution else None + jira_now = parse_datetime(parsed['issue']['fields']['updated']) + + if findings: + for finding in findings: + jira_helper.process_resolution_from_jira(finding, resolution_id, resolution_name, assignee_name, jira_now) if parsed.get('webhookEvent') == 'comment_created': """ @@ -152,16 +161,29 @@ def webhook(request, secret=None): jissue = get_object_or_404(JIRA_Issue, jira_id=jid) logging.info("Received issue comment for {}".format(jissue.jira_key)) logger.debug('jissue: %s', vars(jissue)) + + jira_usernames = JIRA_Instance.objects.values_list('username', flat=True) + for jira_userid in jira_usernames: + # logger.debug('incoming username: %s jira config username: %s', commentor.lower(), jira_userid.lower()) + if jira_userid.lower() == commentor.lower(): + logger.debug('skipping incoming JIRA comment as the user id of the comment in JIRA (%s) matches the JIRA username in DefectDojo (%s)', commentor.lower(), jira_userid.lower()) + return HttpResponse('') + break + + findings = None if jissue.finding: + findings = [jissue.finding] + create_notification(event='other', title='JIRA incoming comment - %s' % (jissue.finding), url=reverse("view_finding", args=(jissue.finding.id, )), icon='check') + elif jissue.finding_group: + findings = [jissue.finding_group.findings.all()] + create_notification(event='other', title='JIRA incoming comment - %s' % (jissue.finding), url=reverse("view_finding_group", args=(jissue.finding_group.id, )), icon='check') + elif jissue.engagement: + return HttpResponse('Comment for engagement ignored') + else: + raise Http404('No finding or engagement found for JIRA issue {}'.format(jissue.jira_key)) + + for finding in findings: # logger.debug('finding: %s', vars(jissue.finding)) - jira_usernames = JIRA_Instance.objects.values_list('username', flat=True) - for jira_userid in jira_usernames: - # logger.debug('incoming username: %s jira config username: %s', commentor.lower(), jira_userid.lower()) - if jira_userid.lower() == commentor.lower(): - logger.debug('skipping incoming JIRA comment as the user id of the comment in JIRA (%s) matches the JIRA username in DefectDojo (%s)', commentor.lower(), jira_userid.lower()) - return HttpResponse('') - break - finding = jissue.finding new_note = Notes() new_note.entry = '(%s (%s)): %s' % (commentor_display_name, commentor, comment_text) new_note.author, created = User.objects.get_or_create(username='JIRA') @@ -170,11 +192,6 @@ def webhook(request, secret=None): finding.jira_issue.jira_change = timezone.now() finding.jira_issue.save() finding.save() - create_notification(event='other', title='JIRA incoming comment - %s' % (jissue.finding), url=reverse("view_finding", args=(jissue.finding.id, )), icon='check') - elif jissue.engagement: - return HttpResponse('Comment for engagement ignored') - else: - raise Http404('No finding or engagement found for JIRA issue {}'.format(jissue.jira_key)) if parsed.get('webhookEvent') not in ['comment_created', 'jira:issue_updated']: logger.info('Unrecognized JIRA webhook event received: {}'.format(parsed.get('webhookEvent'))) diff --git a/dojo/management/commands/auto_delete_engagements.py b/dojo/management/commands/auto_delete_engagements.py new file mode 100644 index 00000000000..91bd5dc7fd8 --- /dev/null +++ b/dojo/management/commands/auto_delete_engagements.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand +from dojo.cb_utils import auto_delete_engagements + +""" +This command will iterate over engagements and delete them if they match required criteria +""" + + +class Command(BaseCommand): + help = 'Launch with no argument.' + + def handle(self, *args, **options): + auto_delete_engagements() diff --git a/dojo/management/commands/jira_status_reconciliation.py b/dojo/management/commands/jira_status_reconciliation.py index b65cc820eb3..2a587d639a3 100644 --- a/dojo/management/commands/jira_status_reconciliation.py +++ b/dojo/management/commands/jira_status_reconciliation.py @@ -207,7 +207,7 @@ class Command(BaseCommand): """ help = 'Reconcile finding status with JIRA issue status, stdout will contain semicolon seperated CSV results. \ - Risk Accepted findings are skipped.' + Risk Accepted findings are skipped. Findings created before 1.14.0 are skipped.' mode_help = \ '- reconcile: (default)reconcile any differences in status between Defect Dojo and JIRA, will look at the latest status change timestamp in both systems to determine which one is the correct status' \ diff --git a/dojo/models.py b/dojo/models.py index 99b78fdf2da..d2b121688b3 100755 --- a/dojo/models.py +++ b/dojo/models.py @@ -1,6 +1,7 @@ import base64 import hashlib import logging +from operator import itemgetter import os import re from uuid import uuid4 @@ -30,6 +31,7 @@ from tagulous.models import TagField import tagulous.admin from django_jsonfield_backport.models import JSONField +from itertools import groupby import hyperlink from cvss import CVSS3 @@ -1924,6 +1926,17 @@ def get_number_severity(severity): else: return 5 + @staticmethod + def get_severity(num_severity): + severities = {0: 'Info', 1: 'Low', 2: 'Medium', 3: 'High', 4: 'Critical'} + logger.debug(severities.keys()) + logger.debug(num_severity in severities.keys()) + if num_severity in severities.keys(): + logger.debug('returning severity: %s', severities[num_severity]) + return severities[num_severity] + + return None + def __str__(self): return self.title @@ -2024,11 +2037,27 @@ def has_jira_issue(self): import dojo.jira_link.helper as jira_helper return jira_helper.has_jira_issue(self) + @cached_property + def finding_group(self): + return self.finding_group_set.all().first() + + @cached_property + def has_jira_group_issue(self): + if not self.has_finding_group: + return False + + import dojo.jira_link.helper as jira_helper + return jira_helper.has_jira_issue(self.finding_group) + @property def has_jira_configured(self): import dojo.jira_link.helper as jira_helper return jira_helper.has_jira_configured(self) + @cached_property + def has_finding_group(self): + return self.finding_group is not None + def long_desc(self): long_desc = '' long_desc += '*' + self.title + '*\n\n' @@ -2258,6 +2287,90 @@ def get_breadcrumbs(self): return bc +class Finding_Group(TimeStampedModel): + name = models.CharField(max_length=255, blank=False, null=False) + test = models.ForeignKey(Test, on_delete=models.CASCADE) + findings = models.ManyToManyField(Finding) + creator = models.ForeignKey(Dojo_User, on_delete=models.CASCADE) + + def __str__(self): + return self.name + + @property + def has_jira_issue(self): + import dojo.jira_link.helper as jira_helper + return jira_helper.has_jira_issue(self) + + @cached_property + def severity(self): + if not self.findings.all(): + return None + max_number_severity = max([Finding.get_number_severity(find.severity) for find in self.findings.all()]) + logger.debug('MAX:%s', max_number_severity) + return Finding.get_severity(max_number_severity) + + @cached_property + def components(self): + # Using defaultdict() + groupby() + # Convert list of tuples to dictionary value lists + component_tuples = set([(find.component_name, find.component_version) for find in self.findings.all()]) + components = dict((k, [v[1] for v in itr]) for k, itr in groupby( + component_tuples, itemgetter(0))) + return ','.join([key + ':' + ', '.join(value) for key, value in components.items() if key and value]) + + @property + def age(self): + if not self.findings.all(): + return None + + return max([find.age for find in self.findings.all()]) + + @cached_property + def sla_days_remaining_internal(self): + if not self.findings.all(): + return None + + return min([find.sla_days_remaining() for find in self.findings.all()]) + + def sla_days_remaining(self): + return self.sla_days_remaining_internal + + def sla_deadline(self): + if not self.findings.all(): + return None + + return min([find.sla_deadline() for find in self.findings.all()]) + + # def cves(self): + # return ', '.join([find.cve for find in self.findings.all() if find.cve is not None]) + + def status(self): + if not self.findings.all(): + return None + + if any([find.active for find in self.findings.all()]): + return 'Active' + + if all([find.is_Mitigated for find in self.findings.all()]): + return 'Mitigated' + + return 'Inactive' + + @cached_property + def mitigated(self): + return all([find.mitigated is not None for find in self.findings.all()]) + + def get_sla_start_date(self): + return min([find.sla_deadline() for find in self.findings.all()]) + + def get_absolute_url(self): + from django.urls import reverse + return reverse('view_test', args=[str(self.test.id)]) + + class Meta: + ordering = ['id'] + + class Finding_Template(models.Model): title = models.TextField(max_length=1000) cwe = models.IntegerField(default=None, null=True, blank=True) @@ -2580,10 +2693,10 @@ class JIRA_Instance(models.Model): choices=default_issue_type_choices, default='Bug', help_text='You can define extra issue types in settings.py') - issue_template = models.CharField(max_length=255, + issue_template_dir = models.CharField(max_length=255, null=True, blank=True, - help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.') + help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.') epic_name_id = models.IntegerField(help_text="To obtain the 'Epic name id' visit https:///rest/api/2/field and search for Epic Name. Copy the number out of cf[number] and paste it here.") open_status_key = models.IntegerField(verbose_name="Reopen Transition ID", help_text="Transition ID to Re-Open JIRA issues, visit https:///rest/api/latest/issue//transitions?expand=transitions.fields to find the ID for your JIRA instance") close_status_key = models.IntegerField(verbose_name="Close Transition ID", help_text="Transition ID to Close JIRA issues, visit https:///rest/api/latest/issue//transitions?expand=transitions.fields to find the ID for your JIRA instance") @@ -2609,6 +2722,7 @@ def __str__(self): return self.configuration_name + " | " + self.url + " | " + self.username def get_priority(self, status): + logger.debug('get_priority for: %s', status) if status == 'Info': return self.info_mapping_severity elif status == 'Low': @@ -2655,10 +2769,10 @@ class JIRA_Project(models.Model): null=True, blank=True, on_delete=models.CASCADE) project_key = models.CharField(max_length=200, blank=True) product = models.ForeignKey(Product, on_delete=models.CASCADE, null=True) - issue_template = models.CharField(max_length=255, + issue_template_dir = models.CharField(max_length=255, null=True, blank=True, - help_text='Choose a Django template used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira-description.tpl.') + help_text='Choose the folder containing the Django templates used to render the JIRA issue description. These are stored in dojo/templates/issue-trackers. Leave empty to use the default jira_full templates.') engagement = models.OneToOneField(Engagement, on_delete=models.CASCADE, null=True, blank=True) component = models.CharField(max_length=200, blank=True) push_all_issues = models.BooleanField(default=False, blank=True, @@ -2710,6 +2824,7 @@ class JIRA_Issue(models.Model): jira_key = models.CharField(max_length=200) finding = models.OneToOneField(Finding, null=True, blank=True, on_delete=models.CASCADE) engagement = models.OneToOneField(Engagement, null=True, blank=True, on_delete=models.CASCADE) + finding_group = models.OneToOneField(Finding_Group, null=True, blank=True, on_delete=models.CASCADE) jira_creation = models.DateTimeField(editable=True, null=True, @@ -2720,6 +2835,16 @@ class JIRA_Issue(models.Model): verbose_name="Jira last update", help_text="The date the linked Jira issue was last modified.") + def set_obj(self, obj): + if type(obj) == Finding: + self.finding = obj + elif type(obj) == Finding_Group: + self.finding_group = obj + elif type(obj) == Engagement: + self.engagement = obj + else: + raise ValueError('unknown objec type whiel creating JIRA_Issue: %s', to_str_typed(obj)) + def __str__(self): text = "" if self.finding: @@ -3378,7 +3503,7 @@ def enable_disable_auditlog(enable=True): auditlog.unregister(Cred_User) -from dojo.utils import get_system_setting +from dojo.utils import get_system_setting, to_str_typed enable_disable_auditlog(enable=get_system_setting('enable_auditlog')) # on startup choose safe to retrieve system settiung) tagulous.admin.register(Product.tags) diff --git a/dojo/notifications/helper.py b/dojo/notifications/helper.py index 64cd4c7872c..dab2ad332d4 100644 --- a/dojo/notifications/helper.py +++ b/dojo/notifications/helper.py @@ -40,6 +40,10 @@ def create_notification(event=None, **kwargs): if not product and 'finding' in kwargs: product = kwargs['finding'].test.engagement.product + if not product and 'obj' in kwargs: + from dojo.utils import get_product + product = get_product(kwargs['obj']) + # notifications are made synchronous again due to serialization bug in django-tagulous # see https://github.com/DefectDojo/django-DefectDojo/issues/3677 # kwargs = convert_kwargs_if_async(**kwargs) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index b3fadda1caa..10e84a5fc56 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -179,7 +179,8 @@ # You need to have wkhtmltopdf installed on your system to generate PDF reports DD_FEATURE_REPORTS_PDF_LIST=(bool, False), - DD_JIRA_TEMPLATE_DIR=(str, 'dojo/templates/issue-trackers'), + DD_FEATURE_FINDING_GROUPS=(bool, False), + DD_JIRA_TEMPLATE_ROOT=(str, 'dojo/templates/issue-trackers'), DD_TEMPLATE_DIR_PREFIX=(str, 'dojo/templates/') ) @@ -1058,5 +1059,6 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param USE_L10N = True -JIRA_TEMPLATE_DIR = env('DD_JIRA_TEMPLATE_DIR') +FEATURE_FINDING_GROUPS = env('DD_FEATURE_FINDING_GROUPS') +JIRA_TEMPLATE_ROOT = env('DD_JIRA_TEMPLATE_ROOT') TEMPLATE_DIR_PREFIX = env('DD_TEMPLATE_DIR_PREFIX') diff --git a/dojo/templates/dojo/delete_finding_group.html b/dojo/templates/dojo/delete_finding_group.html new file mode 100644 index 00000000000..17b8b29a763 --- /dev/null +++ b/dojo/templates/dojo/delete_finding_group.html @@ -0,0 +1,28 @@ +{% extends "base.html" %} +{% block content %} +

Delete Finding Group {{ finding_group }}

+

+ Deleting this Finding Group will NOT remove any findings inside this group. + A Finding Group is just a thin wrapper around a set of findings. +

+
+
+

Danger Zone

+
+ {% if rels|length > 1 %} +
    {{ rels|unordered_list }}
+ {% else %} +

No relationships found.

+ {% endif %} +
+ {% csrf_token %} + {{ form }} + +
+ +
+
+
+
+
+{% endblock %} diff --git a/dojo/templates/dojo/edit_finding.html b/dojo/templates/dojo/edit_finding.html index 4b577d81c7f..4cdc0c9eb4a 100644 --- a/dojo/templates/dojo/edit_finding.html +++ b/dojo/templates/dojo/edit_finding.html @@ -77,7 +77,10 @@

JIRA

- {% if finding.has_jira_issue %} + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_issue_url }} (group) + {% elif finding.has_jira_issue %} {{ finding | jira_issue_url }} {% else %} diff --git a/dojo/templates/dojo/findings_list_snippet.html b/dojo/templates/dojo/findings_list_snippet.html index 696c9fea5b0..1561613c9af 100644 --- a/dojo/templates/dojo/findings_list_snippet.html +++ b/dojo/templates/dojo/findings_list_snippet.html @@ -183,6 +183,9 @@

JIRA Change {% endif %} {% endif%} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + Group + {% endif %} {% if show_product_column and product_tab is None %} {% dojo_sort request 'Product' 'test__engagement__product__name'%} {% endif %} @@ -403,19 +406,39 @@

{% if system_settings.enable_jira %} {% if jira_project and product_tab or not product_tab %} - {% if finding.has_jira_issue %} + {% if finding.has_jira_group_issue %} + {{finding.finding_group | jira_key}} + {% elif finding.has_jira_issue %} {{finding | jira_key}} {% endif %} - {{ finding | jira_creation | timesince }} + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_creation | timesince }} + {% else %} + {{ finding | jira_creation | timesince }} + {% endif %} - {{ finding | jira_change | timesince }} + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_change | timesince }} + {% else %} + {{ finding | jira_change | timesince }} + {% endif %} {% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + + {% if finding.has_finding_group %} + Y + {% else %} + N + {% endif %} + + {% endif %} {% if show_product_column and product_tab is None %} { "data": "jira_change" }, {% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + { "data": "grouped" }, + {% endif %} {% if show_product_column and product_tab is None %} { "data": "product" }, {% endif %} diff --git a/dojo/templates/dojo/view_finding.html b/dojo/templates/dojo/view_finding.html index 96ff5d37461..163532f286f 100755 --- a/dojo/templates/dojo/view_finding.html +++ b/dojo/templates/dojo/view_finding.html @@ -476,6 +476,9 @@

{% if finding.github_conf_new or finding.github_issue %} GitHub {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + Group + {% endif %} {% if finding.file_path %} @@ -515,30 +518,38 @@

{% endif %} - {% if finding.has_jira_configured or finding.jira_issue %} + {% if finding.has_jira_configured or finding.has_jira_issue or finding.has_jira_group_issue %} - {% if finding.jira_issue %} - {{ finding | jira_key }} - - - {% else %} - {% if can_be_pushed_to_jira %} - None - - {% comment %} - - {% endcomment %} + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_key }} + {% endif %} + {% if finding.has_jira_issue %} + {{ finding | jira_key }} + + {% else %} - - + {% if can_be_pushed_to_jira %} + {% if not finding.has_jira_group_issue %} + None + + {% comment %} + + {% endcomment %} + {% endif %} + {% else %} + + + {% endif %} {% endif %} - {% endif %} - {% if finding.jira_issue %} + {% if finding.has_jira_group_issue %} +
{{ finding.finding_group.jira_issue.jira_change|naturalday }}
+ {% elif finding.jira_issue %}
{{ finding.jira_issue.jira_change|naturalday }}
{% endif %} @@ -550,6 +561,16 @@

{% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + + {% if finding.finding_group %} + {{ finding.finding_group.name }} + + {% else %} + N + {% endif %} + + {% endif %}

@@ -1119,16 +1140,12 @@

Credential jqXHR.setRequestHeader('X-CSRFToken', '{{ csrf_token }}'); }, success: function (data, textStatus, jqXHR) { - console.log(textStatus) - console.log(data) }, error: function (request, status, error) { console.log('error') console.log(request.responseText) }, complete: function(e) { - console.log('complete') - console.log(this) location.reload() } }); @@ -1142,6 +1159,10 @@

Credential jira_action(this,'{% url 'finding_unlink_jira' finding.id %}') }); + $(".push_group_to_jira").on('click', function(e) { + jira_action(this, '/finding_group/' + this.dataset.groupId + '/jira/push') + }); + var checkbox_count = 0; function get_checkbox_values() { diff --git a/dojo/templates/dojo/view_test.html b/dojo/templates/dojo/view_test.html index 04ef859bc59..ff2270ac803 100644 --- a/dojo/templates/dojo/view_test.html +++ b/dojo/templates/dojo/view_test.html @@ -255,6 +255,183 @@

Import History ({{ paged_test_imports.total_count }}) {% endif %} + + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} +
+
+
+

Groups ({{ finding_groups|length }}) +
+ + + + + + +
+

+
+
+ +
+ +
+ Experimental feature + {% if finding_groups %} + + + + + + + + + + + + {% if system_settings.enable_finding_sla %} + + {% endif %} + + + {% if system_settings.enable_jira %} + {% if jira_project and product_tab or not product_tab %} + + + + {% endif %} + {% endif %} + + + + {% for group in finding_groups %} + + + + + + + + + + {% if system_settings.enable_finding_sla %} + + {% endif %} + + + {% if system_settings.enable_jira %} + {% if jira_project %} + + + + + + + + {% endif %} + {% endif %} + + {% endfor %} + +
SeverityNameFindingsCVEsComponentsDateAgeSLACreatorStatusJiraJira AgeJira Change
+
+
+    + +
+
+ 		+ + {{ group.severity }} + + {{ group.name|truncatechars_html:60 }} + + {{ group.findings.all|length }} + + + {% for find in group.findings.all %}{% if find.cve%}{{ find.cve }}{% if not forloop.last %},{% endif %}{% endif %}{% endfor %} + + {{ group.components }} + {{ group.created }}{{ group.age }} + {{ group|finding_sla }} + + {% if group.creator.get_full_name and group.creator.get_full_name.strip %} + {{ group.creator.get_full_name }} + {% else %} + {{ group.creator }} + {% endif %} + {{ group.status }} + {% if group.jira_issue %} + {{ group | jira_key }} + + + {% else %} + None + + {% comment %} + + {% endcomment %} + {% endif %} + + {{ group | jira_creation | timesince }} + + {{ group | jira_change | timesince }} +
+ {% else %} +
+

No Groups found.

+
+ {% endif %} +
+ {% endif %} +
@@ -329,7 +506,7 @@

Findings ({{findings.total_count}}) {{ test.id|g

{% endif %} {% endif %} -
+
{% if findings %} @@ -433,6 +632,9 @@

Findings ({{findings.total_count}}) {{ test.id|g

{% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + + {% endif %} @@ -638,23 +840,40 @@

Findings ({{findings.total_count}}) {{ test.id|g {% if system_settings.enable_jira %} {% if jira_project and product_tab or not product_tab %} -

- - - - - + + + {% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + + {% endif %} {% endfor %} @@ -668,7 +887,9 @@

Findings ({{findings.total_count}}) {{ test.id|g {% include "dojo/paging_snippet.html" with page=findings prefix='findings' page_size=True %} -
+
+ +

Potential Findings

@@ -749,9 +970,6 @@

Potential Findings

{% include "dojo/paging_snippet.html" with page=stub_findings %}
- - -
@@ -926,6 +1144,9 @@

Files { "data": "jira_change" }, {% endif %} {% endif %} + {% if 'FEATURE_FINDING_GROUPS'|setting_enabled %} + { "data": "grouped" }, + {% endif %} ]; // Filter the list of items to display based on what is shown. @@ -1110,6 +1331,30 @@

Files $('#bulk_edit_menu #id_bulk_risk_accept').prop('checked', false); }) + $('#id_bulk_finding_group').on('click', function (e) { + var checked = this.checked; + $('#bulk_edit_menu #id_bulk_finding_group_create').prop('disabled', !checked); + $('#bulk_edit_menu #id_bulk_finding_group_add').prop('disabled', !checked); + $('#bulk_edit_menu #id_bulk_finding_group_remove').prop('disabled', !checked); + }) + + $('#bulk_edit_menu #id_bulk_finding_group_create').on("click", function (e){ + var checked = this.checked; + $('#bulk_edit_menu #id_bulk_false_p').prop('disabled', checked); + $('#bulk_edit_menu #id_bulk_is_Mitigated').prop('disabled', checked); + $('#bulk_edit_menu #id_bulk_false_p').prop('checked', false); + $('#bulk_edit_menu #id_bulk_is_Mitigated').prop('checked', false); + }) + + $('.finding_group_option').click(function () { + outer = $(this) + $('.finding_group_option').each(function() { + if (!$(this).is(outer)) { + $(this).prop('checked', false); + } + }) + }); + //Ensures dropdown has proper zindex $('.table-responsive').on('show.bs.dropdown', function () { $('.table-responsive').css( "overflow", "inherit" ); @@ -1308,6 +1553,48 @@

Files }); + function jira_action(elem, url) { + $(elem).removeClass().addClass('fa fa-spinner fa-spin') + + $.ajax({ + type: "post", + dataType:'json', + data: '', + context: this, + url: url, + // The ``X-CSRFToken`` evidently can't be set in the + // ``headers`` option, so force it here. + // This method requires jQuery 1.5+. + beforeSend: function (jqXHR, settings) { + // Pull the token out of the DOM. + jqXHR.setRequestHeader('X-CSRFToken', '{{ csrf_token }}'); + }, + success: function (data, textStatus, jqXHR) { + }, + error: function (request, status, error) { + console.log(request.responseText) + }, + complete: function(e) { + location.reload() + } + }); + } + + $(".push_group_to_jira").on('click', function(e) { + jira_action(this, '/finding_group/' + this.dataset.groupId + '/jira/push') + }); + + $(".unlink_group_jira").on('click', function(e) { + jira_action(this, '/finding_group/' + this.dataset.groupId + '/jira/unlink') + }); + + $('[id^=delete-finding-group-menu-]').on('click', function () { + var form_element = "form#" + this.id + "-form"; + $( form_element ).submit(); + }); + + + {% include "dojo/filter_js_snippet.html" %} {% endblock %} diff --git a/dojo/templates/issue-trackers/jira-description-cb-limited.tpl b/dojo/templates/issue-trackers/jira-description-cb-limited.tpl new file mode 100644 index 00000000000..4f3d8ccae40 --- /dev/null +++ b/dojo/templates/issue-trackers/jira-description-cb-limited.tpl @@ -0,0 +1,29 @@ +{% load navigation_tags %} +{% load display_tags %} +{% url 'view_product' finding.test.engagement.product.id as product_url %} +{% url 'view_engagement' finding.test.engagement.id as engagement_url %} +{% url 'view_test' finding.test.id as test_url %} +{% url 'view_finding' finding.id as finding_url %} + +*Defect Dojo link:* {{ finding_url|full_url }} +*Defect Dojo ID:* {{ finding.id }} + +Please refer to https://cloudbees.atlassian.net/wiki/spaces/ENG/pages/999326760/Security+bug+fix+policy for SLA information. +JIRA Due Date field was automatically calculated based on it, if configured in your JIRA screen. + +*Severity:* {{ finding.severity }} +{% if finding.cwe > 0 %} +*CWE:* [CWE-{{ finding.cwe }}|{{ finding.cwe|cwe_url }}] +{% else %} +*CWE:* Unknown +{% endif %} + +{% if finding.cve %} +*CVE:* [{{ finding.cve }}|{{ finding.cve|cve_url }}] +{% else %} +*CVE:* Unknown +{% endif %} + +*Product/Engagement/Test:* [{{ finding.test.engagement.product.name }}|{{ product_url|full_url }}] / [{{ finding.test.engagement.name }}|{{ engagement_url|full_url }}] / [{{ finding.test }}|{{ test_url|full_url }}] + +*Reporter:* [{{ finding.reporter|full_name}} ({{ finding.reporter.email }})|mailto:{{ finding.reporter.email }}] diff --git a/dojo/templates/issue-trackers/jira-description.tpl b/dojo/templates/issue-trackers/jira_full/jira-description.tpl similarity index 88% rename from dojo/templates/issue-trackers/jira-description.tpl rename to dojo/templates/issue-trackers/jira_full/jira-description.tpl index c25617d0308..6b686f107d2 100644 --- a/dojo/templates/issue-trackers/jira-description.tpl +++ b/dojo/templates/issue-trackers/jira_full/jira-description.tpl @@ -4,9 +4,12 @@ {% url 'view_engagement' finding.test.engagement.id as engagement_url %} {% url 'view_test' finding.test.id as test_url %} {% url 'view_finding' finding.id as finding_url %} -*Title*: [{{ finding.title|jiraencode}}|{{ finding_url|full_url }}] *Defect Dojo link:* {{ finding_url|full_url }} +*Defect Dojo ID:* {{ finding.id }} + +Please refer to https://cloudbees.atlassian.net/wiki/spaces/ENG/pages/999326760/Security+bug+fix+policy for SLA information. +JIRA Due Date field was automatically calculated based on it, if configured in your JIRA screen. *Severity:* {{ finding.severity }} {% if finding.cwe > 0 %} @@ -29,12 +32,14 @@ *Commit hash:* {{ finding.test.engagement.commit_hash }} +{% if finding.endpoints.all %} *Systems/Endpoints*: {% for endpoint in finding.endpoints.all %} * {{ endpoint }}{% endfor %} {% comment %} we leave the endfor at the same line to avoid double line breaks i.e. too many blank lines {% endcomment %} +{%endif%} {% if finding.component_name %} @@ -63,6 +68,4 @@ Sink Object: {{ finding.sast_sink_object }} *References*: {{ finding.references }} -*Defect Dojo ID:* {{ finding.id }} - *Reporter:* [{{ finding.reporter|full_name}} ({{ finding.reporter.email }})|mailto:{{ finding.reporter.email }}] diff --git a/dojo/templates/issue-trackers/jira_full/jira-finding-group-description.tpl b/dojo/templates/issue-trackers/jira_full/jira-finding-group-description.tpl new file mode 100644 index 00000000000..93a1cd63c7a --- /dev/null +++ b/dojo/templates/issue-trackers/jira_full/jira-finding-group-description.tpl @@ -0,0 +1,68 @@ +{% load navigation_tags %} +{% load display_tags %} +{% url 'view_finding_group' finding_group.id as finding_group_url %} +{% url 'view_product' finding.test.engagement.product.id as product_url %} +{% url 'view_engagement' finding.test.engagement.id as engagement_url %} +{% url 'view_test' finding.test.id as test_url %} + +A group of Findings has been pushed to JIRA to be investigated and fixed: + +h2. Group +*Group*: [{{ finding_group.name|jiraencode}}|{{ finding_group_url|full_url }}] in [{{ finding_group.test.engagement.product.name|jiraencode }}|{{ product_url|full_url }}] / [{{ finding_group.test.engagement.name|jiraencode }}|{{ engagement_url|full_url }}] / [{{ finding_group.test|stringformat:'s'|jiraencode }}|{{ test_url|full_url }}] + + +|| Severity || CVE || CWE || Component || Version || Title || Status ||{% for finding in finding_group.findings.all %} +| {{finding.severity}} | {% if finding.cve %}[{{finding.cve}}|{{finding.cve|cve_url}}]{% else %}None{% endif %} | [{{finding.cwe}}|{{finding.cwe|cwe_url}}] | {{finding.component_name|jiraencode_component}} | {{finding.component_version}} | [{{ finding.title|jiraencode}}|{{ finding_url|full_url }}] | {{ finding.status }} |{% endfor %} + +*Branch/Tag:* {{ finding_group.test.engagement.branch_tag }} + +*BuildID:* {{ finding_group.test.engagement.build_id }} + +*Commit hash:* {{ finding_group.test.engagement.commit_hash }} + + +{% for finding in finding_group.findings.all %} +{% url 'view_finding' finding.id as finding_url %} + +h1. Findings + +h3. [{{ finding.title|jiraencode}}|{{ finding_url|full_url }}] +*Defect Dojo link:* {{ finding_url|full_url }} ({{ finding.id }}) +*Severity:* {{ finding.severity }} +{% if finding.cwe > 0 %}*CWE:* [CWE-{{ finding.cwe }}|{{ finding.cwe|cwe_url }}]{% else %}*CWE:* Unknown{% endif %} +{% if finding.cve %}*CVE:* [{{ finding.cve }}|{{ finding.cve|cve_url }}]{% else %}*CVE:* Unknown{% endif %} + +{% if finding.endpoints.all %} +*Systems/Endpoints*: +{% for endpoint in finding.endpoints.all %} +* {{ endpoint }}{% endfor %} +{%endif%} + +{% if finding.component_name %} +Vulnerable Component: {{finding.component_name }} - {{ finding.component_version }} + +{% endif %} +{% if finding.sast_source_object %} +Source Object: {{ finding.sast_source_object }} +Source File: {{ finding.sast_source_file_path }} +Source Line: {{ finding.sast_source_line }} +Sink Object: {{ finding.sast_sink_object }} +{% endif %} + +*Description*: +{{ finding.description }} + +*Mitigation*: +{{ finding.mitigation }} + +*Impact*: +{{ finding.impact }} + +*Steps to reproduce*: +{{ finding.steps_to_reproduce }} + +*References*: +{{ finding.references }} + +*Reporter:* [{{ finding.reporter|full_name}} ({{ finding.reporter.email }})|mailto:{{ finding.reporter.email }}] +{% endfor %} \ No newline at end of file diff --git a/dojo/templates/issue-trackers/jira-description-limited.tpl b/dojo/templates/issue-trackers/jira_limited/jira-description.tpl similarity index 88% rename from dojo/templates/issue-trackers/jira-description-limited.tpl rename to dojo/templates/issue-trackers/jira_limited/jira-description.tpl index 1e5f4400cb0..862a174ea8a 100644 --- a/dojo/templates/issue-trackers/jira-description-limited.tpl +++ b/dojo/templates/issue-trackers/jira_limited/jira-description.tpl @@ -5,8 +5,7 @@ {% url 'view_test' finding.test.id as test_url %} {% url 'view_finding' finding.id as finding_url %} -*Defect Dojo link:* {{ finding_url|full_url }} -*Defect Dojo ID:* {{ finding.id }} +*Defect Dojo link:* {{ finding_url|full_url }} ({{ finding.id }}) *Product/Engagement/Test:* [{{ finding.test.engagement.product.name }}|{{ product_url|full_url }}] / [{{ finding.test.engagement.name }}|{{ engagement_url|full_url }}] / [{{ finding.test }}|{{ test_url|full_url }}] diff --git a/dojo/templates/issue-trackers/jira_limited/jira-finding-group-description.tpl b/dojo/templates/issue-trackers/jira_limited/jira-finding-group-description.tpl new file mode 100644 index 00000000000..2c0996b9aa3 --- /dev/null +++ b/dojo/templates/issue-trackers/jira_limited/jira-finding-group-description.tpl @@ -0,0 +1,20 @@ +{% load navigation_tags %} +{% load display_tags %} +{% url 'view_finding_group' finding_group.id as finding_group_url %} +{% url 'view_product' finding.test.engagement.product.id as product_url %} +{% url 'view_engagement' finding.test.engagement.id as engagement_url %} +{% url 'view_test' finding.test.id as test_url %} + +A group of Findings has been pushed to JIRA to be investigated and fixed: + +*Group*: [{{ finding_group.name|jiraencode}}|{{ finding_group_url|full_url }}] in [{{ finding_group.test.engagement.product.name|jiraencode }}|{{ product_url|full_url }}] / [{{ finding_group.test.engagement.name|jiraencode }}|{{ engagement_url|full_url }}] / [{{ finding_group.test|stringformat:'s'|jiraencode }}|{{ test_url|full_url }}] + +Findings: +{% for finding in finding_group.findings.all %} +- [{{ finding.title|jiraencode}}|{{ finding_url|full_url }}]{% endfor %} + +*Branch/Tag:* {{ finding_group.test.engagement.branch_tag }} + +*BuildID:* {{ finding_group.test.engagement.build_id }} + +*Commit hash:* {{ finding_group.test.engagement.commit_hash }} diff --git a/dojo/templatetags/display_tags.py b/dojo/templatetags/display_tags.py index 5ced42867a3..2f3aab71939 100644 --- a/dojo/templatetags/display_tags.py +++ b/dojo/templatetags/display_tags.py @@ -775,6 +775,15 @@ def jiraencode(value): return value.replace("|", "").replace("@", "") +@register.filter +def jiraencode_component(value): + if not value: + return value + # component names can be long and won't wrap causing everything to look messy + # add some spaces around semicolon + return value.replace("|", "").replace(":", " : ").replace("@", " @ ").replace("?", " ? ").replace("#", " # ") + + @register.filter def jira_project(obj, use_inheritance=True): return jira_helper.get_jira_project(obj, use_inheritance) diff --git a/dojo/test/views.py b/dojo/test/views.py index 53642989ee1..f6da2990514 100644 --- a/dojo/test/views.py +++ b/dojo/test/views.py @@ -121,6 +121,8 @@ def view_test(request, tid): product_tab.setEngagement(test.engagement) jira_project = jira_helper.get_jira_project(test) + finding_groups = test.finding_group_set.all().prefetch_related('findings', 'jira_issue') + bulk_edit_form = FindingBulkUpdateForm(request.GET) google_sheets_enabled = system_settings.enable_google_sheets @@ -180,6 +182,7 @@ def view_test(request, tid): 'sheet_url': sheet_url, 'bulk_edit_form': bulk_edit_form, 'paged_test_imports': paged_test_imports, + 'finding_groups': finding_groups, }) @@ -187,7 +190,7 @@ def prefetch_for_findings(findings): prefetched_findings = findings if isinstance(findings, QuerySet): # old code can arrive here with prods being a list because the query was already executed prefetched_findings = prefetched_findings.select_related('reporter') - prefetched_findings = prefetched_findings.prefetch_related('jira_issue') + prefetched_findings = prefetched_findings.prefetch_related('jira_issue__jira_project__jira_instance') prefetched_findings = prefetched_findings.prefetch_related('test__test_type') prefetched_findings = prefetched_findings.prefetch_related('test__engagement__jira_project__jira_instance') prefetched_findings = prefetched_findings.prefetch_related('test__engagement__product__jira_project_set__jira_instance') @@ -204,6 +207,7 @@ def prefetch_for_findings(findings): prefetched_findings = prefetched_findings.annotate(mitigated_endpoint_count=Count('endpoint_status__id', filter=Q(endpoint_status__mitigated=True))) prefetched_findings = prefetched_findings.prefetch_related('test__engagement__product__authorized_users') prefetched_findings = prefetched_findings.prefetch_related('test__engagement__product__prod_type__authorized_users') + prefetched_findings = prefetched_findings.prefetch_related('finding_group_set') else: logger.debug('unable to prefetch because query was already executed') diff --git a/dojo/unittests/test_jira_config_product.py b/dojo/unittests/test_jira_config_product.py index e905f3a240e..40746222480 100644 --- a/dojo/unittests/test_jira_config_product.py +++ b/dojo/unittests/test_jira_config_product.py @@ -61,10 +61,10 @@ def add_jira_instance(self, data, jira_mock): def test_add_jira_instance(self): response, jira_instance = self.add_jira_instance(self.data_jira_instance) - def test_add_jira_instance_with_issue_template(self): + def test_add_jira_instance_with_issue_template_dir(self): # make sure we get no error when specifying template data = self.data_jira_instance.copy() - data['issue_template'] = 'issue-trackers/jira-description.tpl' + data['issue_template_dir'] = 'issue-trackers/jira_full' response, jira_instance = self.add_jira_instance(data) # no mock so we can assert the exception raised diff --git a/dojo/unittests/test_jira_template.py b/dojo/unittests/test_jira_template.py index 5e825378d67..ff8d167d3f6 100644 --- a/dojo/unittests/test_jira_template.py +++ b/dojo/unittests/test_jira_template.py @@ -16,29 +16,29 @@ def __init__(self, *args, **kwargs): def setUp(self): self.system_settings(enable_jira=True) - def test_get_jira_issue_template_from_project(self): + def test_get_jira_issue_template_dir_from_project(self): product = Product.objects.get(id=1) jira_project = jira_helper.get_jira_project(product) # filepathfield contains full path - jira_project.issue_template = 'issue-trackers/1-jira-description-limited.tpl' + jira_project.issue_template_dir = 'issue-trackers/jira_full_extra' jira_project.save() - self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/1-jira-description-limited.tpl') + self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/jira_full_extra/jira-description.tpl') - def test_get_jira_issue_template_from_instance(self): + def test_get_jira_issue_template_dir_from_instance(self): product = Product.objects.get(id=1) jira_project = jira_helper.get_jira_project(product) - jira_project.issue_template = None + jira_project.issue_template_dir = None jira_project.save() - self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/jira-description.tpl') + self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/jira_full/jira-description.tpl') - def test_get_jira_project_and_instance_no_issue_template(self): + def test_get_jira_project_and_instance_no_issue_template_dir(self): product = Product.objects.get(id=1) jira_project = jira_helper.get_jira_project(product) - jira_project.issue_template = None + jira_project.issue_template_dir = None jira_project.save() jira_instance = jira_helper.get_jira_instance(product) - jira_instance.issue_template = None + jira_instance.issue_template_dir = None jira_instance.save() # no template should return default - self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/jira-description.tpl') + self.assertEqual(jira_helper.get_jira_issue_template(product), 'issue-trackers/jira_full/jira-description.tpl') diff --git a/dojo/urls.py b/dojo/urls.py index 6bba1765872..dde00b6fef7 100755 --- a/dojo/urls.py +++ b/dojo/urls.py @@ -37,6 +37,7 @@ from dojo.endpoint.urls import urlpatterns as endpoint_urls from dojo.engagement.urls import urlpatterns as eng_urls from dojo.finding.urls import urlpatterns as finding_urls +from dojo.finding_group.urls import urlpatterns as finding_group_urls from dojo.home.urls import urlpatterns as home_urls from dojo.metrics.urls import urlpatterns as metrics_urls from dojo.product.urls import urlpatterns as prod_urls @@ -136,6 +137,7 @@ ur += endpoint_urls ur += eng_urls ur += finding_urls +ur += finding_group_urls ur += home_urls ur += metrics_urls ur += prod_urls diff --git a/dojo/user/helper.py b/dojo/user/helper.py index c6ed123e7c6..f3399083991 100644 --- a/dojo/user/helper.py +++ b/dojo/user/helper.py @@ -3,7 +3,7 @@ from django.core.exceptions import PermissionDenied import functools from django.shortcuts import get_object_or_404 -from dojo.models import Finding, Test, Engagement, Product, Endpoint, Product_Type, \ +from dojo.models import Finding, Finding_Group, Test, Engagement, Product, Endpoint, Product_Type, \ Risk_Acceptance from crum import get_current_user @@ -76,6 +76,7 @@ def _wrapped(request, *args, **kwargs): # print('user is authorized for: ', obj) # Django doesn't seem to easily support just passing on the original positional parameters # so we resort to explicitly putting lookup_value here (which is for example the 'fid' parameter) + print('view_func params', lookup_value, *args, **kwargs) return view_func(request, lookup_value, *args, **kwargs) return _wrapped @@ -92,6 +93,8 @@ def check_auth_users_list(user, obj): elif isinstance(obj, Finding): is_authorized = user in obj.test.engagement.product.authorized_users.all() is_authorized = user in obj.test.engagement.product.prod_type.authorized_users.all() or is_authorized + elif isinstance(obj, Finding_Group): + return check_auth_users_list(obj.test) elif isinstance(obj, Test): is_authorized = user in obj.engagement.product.authorized_users.all() is_authorized = user in obj.engagement.product.prod_type.authorized_users.all() or is_authorized diff --git a/dojo/utils.py b/dojo/utils.py index dfd84ad54f0..592c73bea51 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -22,8 +22,8 @@ from django.db.models.query import QuerySet import calendar as tcalendar from dojo.github import add_external_issue_github, update_external_issue_github, close_external_issue_github, reopen_external_issue_github -from dojo.models import Finding, Engagement, Finding_Template, Product, \ - Dojo_User, User, System_Settings, Notifications, Endpoint, Benchmark_Type, \ +from dojo.models import Finding, Engagement, Finding_Group, Finding_Template, Product, \ + Dojo_User, Test, User, System_Settings, Notifications, Endpoint, Benchmark_Type, \ Language_Type, Languages, Rule from asteval import Interpreter from dojo.notifications.helper import create_notification @@ -1778,7 +1778,13 @@ def _notify(finding, title): continue do_jira_sla_comment = False + jira_issue = None if finding.has_jira_issue: + jira_issue = finding.jira_issue + elif finding.grouped: + jira_issue = finding.finding_group.jira_issue + + if jira_issue: jira_count += 1 jira_instance = jira_helper.get_jira_instance(finding) if jira_instance is not None: @@ -1799,7 +1805,6 @@ def _notify(finding, title): product_jira_sla_comment_enabled )) do_jira_sla_comment = True - jira_issue = finding.jira_issue logger.debug("JIRA issue is {}".format(jira_issue.jira_key)) logger.debug("Finding {} has {} days left to breach SLA.".format(finding.id, sla_age)) @@ -1884,6 +1889,14 @@ def get_object_or_none(klass, *args, **kwargs): return None +def add_success_message_to_response(message): + if get_current_request(): + messages.add_message(get_current_request(), + messages.SUCCESS, + message, + extra_tags='alert-success') + + def add_error_message_to_response(message): if get_current_request(): messages.add_message(get_current_request(), @@ -1896,3 +1909,33 @@ def add_field_errors_to_response(form): if form and get_current_request(): for field, error in form.errors.items(): add_error_message_to_response(error) + + +def to_str_typed(obj): + """ for code that handles multiple types of objects, print not only __str__ but prefix the type of the object""" + return '%s: %s' % (type(obj), obj) + + +def get_product(obj): + logger.debug('getting product for %s:%s', type(obj), obj) + if not obj: + return None + + if type(obj) == Finding or type(obj) == Finding_Group: + return obj.test.engagement.product + + if type(obj) == Test: + return obj.engagement.product + + if type(obj) == Engagement: + return obj.product + + if type(obj) == Product: + return obj + + +def prod_name(obj): + if not obj: + return 'Unknown' + + return get_product(obj).name diff --git a/helm/defectdojo/templates/django-deployment.yaml b/helm/defectdojo/templates/django-deployment.yaml index 8bf8f6c6beb..589b5d8ceb5 100644 --- a/helm/defectdojo/templates/django-deployment.yaml +++ b/helm/defectdojo/templates/django-deployment.yaml @@ -76,6 +76,7 @@ spec: timeoutSeconds: 5 {{- end }} - name: uwsgi + command: ["/entrypoint-uwsgi-cb.sh"] image: '{{ template "django.uwsgi.repository" . }}:{{ .Values.tag }}' imagePullPolicy: {{ .Values.imagePullPolicy }} {{- if .Values.securityContext.enabled }}

Jira ChangeGroup
- {% if finding.has_jira_issue %} - {{finding | jira_key}} - {% endif %} - - {{ finding | jira_creation | timesince }} - - {{ finding | jira_change | timesince }} - + {% if finding.has_jira_group_issue %} + {{finding.finding_group | jira_key}} + {% elif finding.has_jira_issue %} + {{finding | jira_key}} + {% endif %} + + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_creation | timesince }} + {% else %} + {{ finding | jira_creation | timesince }} + {% endif %} + + {% if finding.has_jira_group_issue %} + {{ finding.finding_group | jira_change | timesince }} + {% else %} + {{ finding | jira_change | timesince }} + {% endif %} + + {% if finding.has_finding_group %} + Y + {% else %} + N + {% endif %} +