4.2.5 (#2457)

* Unified async scan timeout
* Allow incomplete scan delete after async scan timeout duration
* Added support for Android SBOM analysis
* Make dependencies unpinned (Address #2458)

mobsf/MobSF/init.py

@@ -18,7 +18,7 @@
 
 logger = logging.getLogger(__name__)
 
-VERSION = '4.2.4'
+VERSION = '4.2.5'
 BANNER = r"""
   __  __       _    ____  _____       _  _    ____  
  |  \/  | ___ | |__/ ___||  ___|_   _| || |  |___ \ 

mobsf/MobSF/settings.py

@@ -341,12 +341,14 @@
         },
     },
 }
+ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
+ASYNC_ANALYSIS_TIMEOUT = int(os.getenv('MOBSF_ASYNC_ANALYSIS_TIMEOUT', '60'))
 Q_CLUSTER = {
     'name': 'scan_queue',
     'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', 3)),
     'recycle': 5,
-    'timeout': 3600,
-    'retry': 3700,
+    'timeout': ASYNC_ANALYSIS_TIMEOUT * 60,
+    'retry': (ASYNC_ANALYSIS_TIMEOUT * 60) + 100,
     'compress': True,
     'label': 'scan_queue',
     'orm': 'default',
@@ -355,7 +357,6 @@
     'ack_failures': True,
 }
 QUEUE_MAX_SIZE = 100
-ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
 MULTIPROCESSING = os.getenv('MOBSF_MULTIPROCESSING')
 JADX_TIMEOUT = int(os.getenv('MOBSF_JADX_TIMEOUT', 1000))
 SAST_TIMEOUT = int(os.getenv('MOBSF_SAST_TIMEOUT', 1000))

mobsf/MobSF/utils.py

@@ -61,6 +61,8 @@
 USERNAME_REGEX = re.compile(r'^\w[\w\-\@\.]{1,35}$')
 GOOGLE_API_KEY_REGEX = re.compile(r'AIza[0-9A-Za-z-_]{35}$')
 GOOGLE_APP_ID_REGEX = re.compile(r'\d{1,2}:\d{1,50}:android:[a-f0-9]{1,50}')
+PKG_REGEX = re.compile(
+    r'package\s+([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*);')
 
 
 class Color(object):

mobsf/MobSF/views/api/api_static_analysis.py

@@ -113,7 +113,6 @@ def api_scan_logs(request):
     return make_api_response({'logs': resp}, 200)
 
 
-
 @request_method(['POST'])
 @csrf_exempt
 def api_tasks(request):

mobsf/MobSF/views/home.py

@@ -7,9 +7,11 @@
 import re
 import shutil
 from pathlib import Path
+from datetime import timedelta
 from wsgiref.util import FileWrapper
 
 from django.conf import settings
+from django.utils.timezone import now
 from django.core.paginator import Paginator
 from django.http import HttpResponse, HttpResponseRedirect
 from django.views.decorators.http import require_http_methods
@@ -541,10 +543,14 @@ def delete_scan(request, api=False):
         if settings.ASYNC_ANALYSIS:
             # Handle Async Tasks
             et = EnqueuedTask.objects.filter(checksum=md5_hash).first()
-            if et and not et.completed_at:
-                # Queue is in progress, cannot delete the task
-                return send_response({
-                    'deleted': 'A scan can only be deleted after it is completed'}, api)
+            if et:
+                max_time_passed = now() - et.created_at > timedelta(
+                    minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
+                if not (et.completed_at or max_time_passed):
+                    # Queue is in progress, cannot delete the task
+                    return send_response(
+                        {'deleted': 'A scan can only be deleted after it is completed'},
+                        api)
         # Delete all related DB entries
         EnqueuedTask.objects.filter(checksum=md5_hash).all().delete()
         RecentScansDB.objects.filter(MD5=md5_hash).delete()
@@ -565,7 +571,7 @@ def delete_scan(request, api=False):
                 os.remove(item_path)
             # Delete related directories
             if is_dir_exists(item_path) and valid_item:
-                shutil.rmtree(item_path)
+                shutil.rmtree(item_path, ignore_errors=True)
         return send_response({'deleted': 'yes'}, api)
     except Exception as exp:
         msg = str(exp)

mobsf/StaticAnalyzer/models.py

@@ -82,6 +82,7 @@ class Meta:
     PLAYSTORE_DETAILS = models.TextField(default={})
     NETWORK_SECURITY = models.TextField(default=[])
     SECRETS = models.TextField(default=[])
+    SBOM = models.TextField(default={})
 
 
 class StaticAnalyzerIOS(models.Model):

mobsf/StaticAnalyzer/views/android/code_analysis.py

@@ -26,6 +26,9 @@
 from mobsf.MalwareAnalyzer.views.android import (
     behaviour_analysis,
 )
+from mobsf.StaticAnalyzer.views.android import (
+    sbom_analysis,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +86,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
         email_n_file = []
         url_n_file = []
         url_list = []
+        sbom = {}
         app_dir = Path(app_dir)
         src = get_android_src_dir(app_dir, typ).as_posix() + '/'
         skp = settings.SKIP_CLASS_PATH
@@ -95,10 +99,17 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
             'match_extensions': {'.java', '.kt'},
             'ignore_paths': skp,
         }
-        # Code Analysis
         sast = SastEngine(options, src)
         # Read data once and pass it to all the analysis
         file_data = sast.read_files()
+
+        # SBOM Analysis
+        sbom = sbom_analysis.sbom(app_dir, file_data)
+        msg = 'Android SBOM Analysis Completed'
+        logger.info(msg)
+        append_scan_status(checksum, msg)
+
+        # Code Analysis
         code_findings = sast.run_rules(file_data, code_rules.as_posix())
         msg = 'Android SAST Completed'
         logger.info(msg)
@@ -186,6 +197,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
             'urls_list': url_list,
             'urls': url_n_file,
             'emails': email_n_file,
+            'sbom': sbom,
         }
         return code_an_dic
     except Exception as exp:

mobsf/StaticAnalyzer/views/android/db_interaction.py

@@ -88,6 +88,7 @@ def get_context_from_db_entry(db_entry: QuerySet) -> dict:
             'playstore_details': python_dict(db_entry[0].PLAYSTORE_DETAILS),
             'secrets': python_list(db_entry[0].SECRETS),
             'logs': get_scan_logs(db_entry[0].MD5),
+            'sbom': python_dict(db_entry[0].SBOM),
         }
         return context
     except Exception:
@@ -161,6 +162,7 @@ def get_context_from_analysis(app_dic,
             'playstore_details': app_dic['playstore'],
             'secrets': code_an_dic['secrets'],
             'logs': get_scan_logs(app_dic['md5']),
+            'sbom': code_an_dic['sbom'],
         }
         return context
     except Exception as exp:
@@ -226,6 +228,7 @@ def save_or_update(update_type,
             'PLAYSTORE_DETAILS': app_dic['playstore'],
             'NETWORK_SECURITY': man_an_dic['network_security'],
             'SECRETS': code_an_dic['secrets'],
+            'SBOM': code_an_dic['sbom'],
         }
         if update_type == 'save':
             db_entry = StaticAnalyzerAndroid.objects.filter(

mobsf/StaticAnalyzer/views/android/sbom_analysis.py

@@ -0,0 +1,78 @@
+# -*- coding: utf_8 -*-
+"""Extract packages from APK."""
+import logging
+
+from mobsf.MobSF.utils import (
+    PKG_REGEX,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def merge_common_packages(items):
+    """Merge common packages."""
+    items = list(items)
+    items.sort()  # Sort items lexicographically
+    merged = []
+    for item in items:
+        if not merged or not item.startswith(merged[-1] + '.'):
+            merged.append(item)
+    return merged
+
+
+def extract_packages(file_data):
+    """Extract package names from file data."""
+    packages = set()
+    try:
+        for item in file_data:
+            # tuple has file path and file content
+            # we are interested in file content's first line
+            pkg = item[1].split('\n')[0]
+            match = PKG_REGEX.search(pkg)
+            if match and match.group(1) != '_COROUTINE':
+                packages.add(match.group(1))
+        packages = merge_common_packages(packages)
+    except Exception:
+        logger.exception('Extracting packages from file data')
+    return sorted(packages)
+
+
+def get_group_name(file_name, group):
+    """Get group and name from file name."""
+    parts = file_name.split('_')
+
+    if parts and len(parts) == 2:
+        group, name = parts[0], parts[1]
+    else:
+        name = file_name.replace('_', '-')
+        if name.startswith('kotlinx-'):
+            group = 'org.jetbrains.kotlinx'
+
+    return group, name
+
+
+def android_sbom(app_dir):
+    """Extract SBOM from files."""
+    sbom = set()
+    for vfile in app_dir.rglob('*.version'):
+        try:
+            dependency = vfile.stem
+            group, name = '', ''
+            version = vfile.read_text().strip() or ''
+            version = 'dynamic' if version.startswith('task') else version
+
+            if '_' in dependency:
+                group, name = get_group_name(dependency, group)
+            sbom.add(f'{group}:{name}@{version}')
+        except Exception:
+            pass
+    return sorted(sbom)
+
+
+def sbom(app_dir, file_data):
+    """Extract SBOM from version files and decompiled source code."""
+    return {
+        'sbom_versioned': android_sbom(app_dir),
+        'sbom_packages': extract_packages(file_data),
+    }

mobsf/StaticAnalyzer/views/android/so.py

@@ -123,6 +123,7 @@ def so_analysis(request, app_dic, rescan, api):
             'urls_list': [],
             'urls': [],
             'emails': [],
+            'sbom': {},
         }
         # Get the strings and metadata from shared object
         get_strings_metadata(

mobsf/StaticAnalyzer/views/common/async_task.py

@@ -52,11 +52,13 @@ def async_analysis(checksum, api, file_name, func, *args, **kwargs):
     recent = RecentScansDB.objects.filter(MD5=checksum)
     scan_completed = recent[0].APP_NAME or recent[0].PACKAGE_NAME
     # Check if the task is updated within the last 60 minutes
-    active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(minutes=60)
+    active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(
+        minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
     # Check if the task is already enqueued within the last 60 minutes
     queued_recently = EnqueuedTask.objects.filter(
         checksum=checksum,
-        created_at__gte=timezone.now() - timedelta(minutes=60),
+        created_at__gte=timezone.now() - timedelta(
+            minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
     ).exists()
 
     # Additional checks on recent queue

mobsf/templates/static_analysis/android_binary_analysis.html

@@ -324,6 +324,12 @@
                   <i class="fab fa-buffer nav-icon"></i>
                   <p>Libraries</p>
                 </a>
+              </li>
+              <li class="nav-item">
+                <a href="#sbom" class="nav-link">
+                  <i class="fa fa-archive nav-icon"></i>
+                  <p>SBOM</p>
+                </a>
               </li>
                <li class="nav-item">
                 <a href="#files" class="nav-link">
@@ -2337,6 +2343,28 @@ <h5 class="description-header">{{ code_analysis.summary.suppressed }}</h5>
       </div>
 </section>
  <!-- ===========================end libraries ================================== -->
+ <a id="sbom" class="anchor"></a>
+ <section class="content">
+   <div class="container-fluid">
+   <div class="row">
+       <div class="col-lg-12">
+       <div class="card">
+         <div class="card-body">
+             <p>
+             <strong><i class="fa fa-archive"></i> SBOM</strong>
+             </p>
+               <div class="list-group">
+                 {% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
+                 {% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
+             </div>
+           </div>
+         </div>
+       </div><!-- /.card -->
+       </div>
+       <!-- end row -->
+       </div>
+ </section>
+  <!-- ===========================end sbom ================================== -->
  <a id="files" class="anchor"></a>
 <section class="content">
   <div class="container-fluid">

mobsf/templates/static_analysis/android_source_analysis.html

@@ -241,6 +241,12 @@
                   <i class="fab fa-buffer nav-icon"></i>
                   <p>Libraries</p>
                 </a>
+              </li>
+              <li class="nav-item">
+                <a href="#sbom" class="nav-link">
+                  <i class="fa fa-archive nav-icon"></i>
+                  <p>SBOM</p>
+                </a>
               </li>
                <li class="nav-item">
                 <a href="#files" class="nav-link">
@@ -1744,6 +1750,28 @@ <h5 class="description-header">{{ code_analysis.summary.suppressed }}</h5>
       </div>
 </section>
  <!-- ===========================end libraries ================================== -->
+ <a id="sbom" class="anchor"></a>
+ <section class="content">
+   <div class="container-fluid">
+   <div class="row">
+       <div class="col-lg-12">
+       <div class="card">
+         <div class="card-body">
+             <p>
+             <strong><i class="fa fa-archive"></i> SBOM</strong>
+             </p>
+               <div class="list-group">
+                 {% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
+                 {% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
+             </div>
+           </div>
+         </div>
+       </div><!-- /.card -->
+       </div>
+       <!-- end row -->
+       </div>
+ </section>
+  <!-- ===========================end sbom ================================== -->
 <a id="files" class="anchor"></a>
 <section class="content">
   <div class="container-fluid">

mobsf/templates/static_analysis/ios_source_analysis.html

@@ -7,7 +7,6 @@
     <!-- DataTables -->
     <link href="{% static "datatables/css/datatables.combined.min.css" %}" rel="stylesheet">
     <link href="{% static "adminlte/plugins/sweetalert2/sweetalert2.min.css" %}" rel="stylesheet">
-
     <style type="text/css" media="print">
         @page { size: landscape; }
         @media print {