Merge branch '2.4' of github.com:MISP/MISP into 2.4

.github/workflows/main.yml

@@ -218,7 +218,7 @@ jobs:
               sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
               . ./venv/bin/activate
               export PYTHONPATH=$PYTHONPATH:./app/files/scripts
-              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara
+              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
               deactivate
 
       - name: Test if apache is working
@@ -260,11 +260,9 @@ jobs:
 
               . ./venv/bin/activate
               pushd PyMISP
-              ls -la .
-              ls -la tests
-              ls -la tests/viper-test-files
-              python tests/testlive_comprehensive.py
-              python tests/test_mispevent.py
+              cp tests/keys.py .
+              python -m pytest -v --durations=0 tests/test_mispevent.py
+              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py

.gitmodules

@@ -17,9 +17,6 @@
 [submodule "app/files/misp-objects"]
 	path = app/files/misp-objects
 	url = https://github.com/MISP/misp-objects
-[submodule "misp-vagrant"]
-	path = misp-vagrant
-	url = https://github.com/MISP/misp-vagrant.git
 [submodule "cti-python-stix2"]
 	path = app/files/scripts/cti-python-stix2
 	url = https://github.com/MISP/cti-python-stix2

.travis.yml

@@ -27,8 +27,9 @@ install:
     - sudo apt-get -y update
     # Install haveged, because Travis lacks entropy.
     - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
-    - sudo pip3 install --upgrade pip setuptools requests pyzmq
+    - sudo pip3 install --upgrade pip setuptools requests
     - sudo pip3 install --upgrade -r requirements.txt
+    - sudo pip3 install --upgrade -r requirements-dev.txt
     - pip3 install --user poetry
     - phpenv rehash
     - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer

INSTALL/INSTALL.sh

@@ -794,6 +794,14 @@ installRNG () {
 kaliUpgrade () {
   debug "Running various Kali upgrade tasks"
   checkAptLock
+  # Fix Missing keys early
+  sudo wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
+  # /!\ The following is a very ugly dependency hack to make php7.4 work on Kali
+  wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  wget http://ftp.debian.org/debian/pool/main/i/icu/libicu67_67.1-7_amd64.deb
+  sudo dpkg -i libicu67_67.1-7_amd64.deb
+  # EOH End-Of-Hack
   sudo DEBIAN_FRONTEND=noninteractive apt update
   sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y
   sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y
@@ -3597,6 +3605,7 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic

INSTALL/INSTALL.sh.sfv

@@ -1,5 +1,5 @@
-; Generated by RHash v1.4.4 on 2023-08-22 at 17:22.35
+; Generated by RHash v1.4.4 on 2023-10-12 at 13:42.08
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160749  17:22.35 2023-08-22 INSTALL.sh
-INSTALL.sh 06BE6B05BBAD5007BDDDB73DBA2F090A3F4552B1 A4A53EB3EC60FFAD773E8E1D76278315B40042E1B2E62971E73D3F66E9327143 98072442A60BE33F9CCF8C205E4CB2A894CB060566ED9CB835DD4B38C6EDD66B2A94ABE860EFEBD9980EE6C1EF4A5B06 EE56B1BF53930F16CCF13B9C308D55E74D52CF65C1BFB03B890E06476A84F30B2C0AF0F488E34A7A22666B3C1F49866598A35B1EB9F3ADE57427DC56E772B7C9
+;       161244  13:42.07 2023-10-12 INSTALL.sh
+INSTALL.sh 8624B1D834A4C958B16DCE32CEAE88C3D1DC15D7 D7B9E370C85B53BBF7B0F81F5C263B6AB2E534F59B40A6499277F39407FF194A EF5B68E9D0D634C2CADD4FD9B1E5C2A93DBD938F488417F67A2B9C87E8867CB0200293A150CDAEDA369E7FDC476EEC2B 1445710924BC029647CC5AA0EBFFD0A3B6DDAF39D26B5A0E951FFFEEF621677C59470F250ECFB6059C9E200951F98D4F21646F152D6F8931438531F9516A8748

INSTALL/INSTALL.sh.sha1

@@ -1 +1 @@
-06be6b05bbad5007bdddb73dba2f090a3f4552b1  INSTALL.sh
+8624b1d834a4c958b16dce32ceae88c3d1dc15d7  INSTALL.sh

INSTALL/INSTALL.sh.sha256

@@ -1 +1 @@
-a4a53eb3ec60ffad773e8e1d76278315b40042e1b2e62971e73d3f66e9327143  INSTALL.sh
+d7b9e370c85b53bbf7b0f81f5c263b6ab2e534f59b40a6499277f39407ff194a  INSTALL.sh

INSTALL/INSTALL.sh.sha384

@@ -1 +1 @@
-98072442a60be33f9ccf8c205e4cb2a894cb060566ed9cb835dd4b38c6edd66b2a94abe860efebd9980ee6c1ef4a5b06  INSTALL.sh
+ef5b68e9d0d634c2cadd4fd9b1e5c2a93dbd938f488417f67a2b9c87e8867cb0200293a150cdaeda369e7fdc476eec2b  INSTALL.sh

INSTALL/INSTALL.sh.sha512

@@ -1 +1 @@
-ee56b1bf53930f16ccf13b9c308d55e74d52cf65c1bfb03b890e06476a84f30b2c0af0f488e34a7a22666b3c1f49866598a35b1eb9f3ade57427dc56e772b7c9  INSTALL.sh
+1445710924bc029647cc5aa0ebffd0a3b6ddaf39d26b5a0e951fffeef621677c59470f250ecfb6059c9e200951f98d4f21646f152d6f8931438531f9516a8748  INSTALL.sh

INSTALL/INSTALL.tpl.sh

@@ -844,6 +844,7 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic

README.md

@@ -1,46 +1,53 @@
 MISP - Threat Intelligence Sharing Platform
 -------------------------------------------
+<img align="right" alt="MISP logo" src="./INSTALL/logos/misp-logo.png"/> 
 
-![logo](./INSTALL/logos/misp-logo.png?raw=true "MISP")
+MISP is an open source software solution for collecting, storing, distributing and sharing cyber security indicators and threats about cyber security incidents analysis and malware analysis. MISP is designed by and for incident analysts, security and ICT professionals or malware reversers to support their day-to-day operations to share structured information efficiently.
+
+The objective of MISP is to foster the sharing of structured information within the security community and abroad. MISP provides functionalities to support the exchange of information but also the consumption of said information by Network Intrusion Detection Systems (NIDS), LIDS but also log analysis tools, SIEMs.
+
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#core-functions">Core functions</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#website--support">Website / Support</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#php-and-misp">PHP and MISP</a><br>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#installation">Installation</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#documentation">Documentation</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#contributing">Contributing</a><br>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#license">License</a>
 
 <table>
 <tr>
   <td>Latest Release</td>
-  <td><a href="https://badge.fury.io/gh/MISP%2FMISP"><img src="https://badge.fury.io/gh/MISP%2FMISP.svg" alt="GitHub version" height="18"></a></td>
-</tr>
-<tr>
-  <td>CI Action</td>
-  <td><a href="https://github.com/MISP/MISP/actions?query=workflow%3Amisp"><img src="https://github.com/MISP/MISP/workflows/misp/badge.svg" /></a></td>
+  <td><a href="https://badge.fury.io/gh/MISP%2FMISP"><img src="https://badge.fury.io/gh/MISP%2FMISP.svg" alt="GitHub version" height="25"></a></td>
+</tr><tr>
+  <td>CI</td>
+  <td><a href="https://github.com/MISP/MISP/actions?query=workflow%3Amisp"><img src="https://img.shields.io/github/actions/workflow/status/MISP/MISP/main.yml?label=test" height="25" /></a></td>
 </tr>
 <tr>
   <td>Gitter</td>
-  <td><a href="https://gitter.im/MISP/MISP?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/MISP/MISP.svg" /></a></td>
+  <td><a href="https://gitter.im/MISP/MISP?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/MISP/MISP.svg" height="25" /></a></td>
 </tr>
 <tr>
-  <td>Twitter</td>
-  <td><a href="https://twitter.com/MISPProject"><img src="https://img.shields.io/twitter/follow/MISPProject.svg?style=social&label=Follow" /></a></td>
-</tr>
+  <td>Mastodon</td>
+  <td><a href="https://misp-community.org/@misp"><img src="https://img.shields.io/badge/follow-@misp-purple" height="25" /></a></td>
+</tr><tr>
 <tr>
+  <td>Twitter</td>
+  <td><a href="https://twitter.com/MISPProject"><img src="https://img.shields.io/badge/follow-@MISPProject-blue" height="25" /></a></td>
+</tr><tr>
   <td>Localization</td>
-  <td><a href="https://crowdin.com/project/misp"><img src="https://badges.crowdin.net/misp/localized.svg" /></a></td>
+  <td><a href="https://crowdin.com/project/misp"><img src="https://badges.crowdin.net/misp/localized.svg" height="25" /></a></td>
 </tr>
 <tr>
-  <td>Contributors</td>
-  <td><img src="https://img.shields.io/github/contributors/MISP/MISP.svg" /></td>
+   <td>Contributors</td>
+  <td><img src="https://img.shields.io/github/contributors/MISP/MISP.svg" height="25" /></td>
+  </tr><tr>
+     <td>License</td>
+  <td><img src="https://img.shields.io/github/license/MISP/MISP.svg" height="25" /></td>
 </tr>
-<tr>
-  <td>License</td>
-  <td><img src="https://img.shields.io/github/license/MISP/MISP.svg" /></td>
-</tr>
-
 </table>
 
-MISP is an open source software solution for collecting, storing, distributing and sharing cyber security indicators and threats about cyber security incidents analysis and malware analysis. MISP is designed by and for incident analysts, security and ICT professionals or malware reversers to support their day-to-day operations to share structured information efficiently.
-
-The objective of MISP is to foster the sharing of structured information within the security community and abroad. MISP provides functionalities to support the exchange of information but also the consumption of said information by Network Intrusion Detection Systems (NIDS), LIDS but also log analysis tools, SIEMs.
-
-MISP, Malware Information Sharing Platform and Threat Sharing, core functionalities are:
-
+Core functions
+------------------
 - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.
 - Automatic **correlation** finding relationships between attributes and indicators from malware, attack campaigns or analysis. The correlation engine includes correlation between attributes and more advanced correlations like Fuzzy hashing correlation (e.g. ssdeep) or CIDR block matching. Correlation can also be enabled or event disabled per attribute.
 - A **flexible data model** where complex [objects](https://www.misp-project.org/objects.html) can be expressed and **linked together to express threat intelligence, incidents or connected elements**.
@@ -73,16 +80,27 @@ A sample event encoded in MISP:
 Website / Support
 ------------------
 
-Checkout the [website](https://www.misp-project.org) for more information about MISP software, standards, tools and communities. 
+Checkout the [website](https://www.misp-project.org) for more information about MISP software, standards, tools and communities.
 
-Information, news and updates are also regularly posted on the [MISP project twitter account](https://twitter.com/MISPProject) or the [news page](https://www.misp-project.org/news/).
+Information, news and updates are also regularly posted on the MISP project [Mastodon account](https://misp-community.org/@misp), [twitter account](https://twitter.com/MISPProject) and [news page](https://www.misp-project.org/news/).
+
+PHP and MISP
+-------------
+MISP currently **requires PHP 7.4**, an end-of-life version of PHP. Because of this it is recommended that you only run MISP on distributions or PHP installs that you know will get security fixes backported, like Red Hat or Debian and derratives.
+
+MISP 3.x, currently in development will support PHP 8.x.
+
+
+Installation
+-------------
+For test- og production installations we recommend you check out the possible options on [misp-project.org/download](https://www.misp-project.org/download/).
 
 Documentation
 -------------
 
 [MISP user-guide (MISP-book)](https://github.com/MISP/misp-book) is available [online](https://www.circl.lu/doc/misp/) or as [PDF](https://www.circl.lu/doc/misp/book.pdf) or as [EPUB](https://www.circl.lu/doc/misp/book.epub) or as [MOBI/Kindle](https://www.circl.lu/doc/misp/book.mobi).
 
-For the installation guide see the [INSTALL](https://github.com/MISP/MISP/tree/2.4/INSTALL) or [download section](https://www.misp-project.org/download/).
+It is also recommended to read the [FAQ](https://github.com/MISP/MISP/wiki/Frequently-Asked-Questions)
 
 Contributing
 ------------

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":177}
+{"major":2, "minor":4, "hotfix":178}

app/Console/Command/DevShell.php

@@ -1,7 +1,11 @@
 <?php
+
+App::uses('ComponentCollection', 'Controller');
+App::uses('RestSearchComponent', 'Controller/Component');
+
 class DevShell extends AppShell {
 
-    public $uses = [];
+    public $uses = ['Attribute', 'Event', 'Object', 'GalaxyCluster', 'Sighting'];
 
     public function cleanFeedDefault() {
         $this->out(__('Massaging the feed metadata file.'));
@@ -46,4 +50,57 @@ public function cleanFeedDefault() {
             }
         }
     }
+
+    public function generateSearchParams()
+    {
+        $fetchFunctionName = [
+            'Attribute' => 'fetchAttributes',
+            'Event' => 'fetchEvents',
+            'Object' => 'fetchObjects',
+            'Sighting' => 'fetchSightings',
+            'GalaxyCluster' => 'fetchGalaxyClusters'
+        ];
+        $collection = new ComponentCollection();
+        $this->RestSearchComponent = $collection->load('RestSearch');
+        $paramArray = $this->RestSearchComponent->paramArray;
+        foreach ($paramArray as $scope => $params) {
+            if (!empty($this->$scope->possibleOptions)) {
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $this->$scope->possibleOptions)));
+            } else {
+                $fileName = $scope === 'Object' ? 'MispObject' : $scope;
+                $code = file_get_contents(APP . 'Model/' . $fileName . '.php');
+                $code = explode("\n", $code);
+                $start = false;
+                $end = false;
+                $analyzedBlock = [];
+                foreach ($code as $lineNumber => $line) {
+                    if (strpos($line, 'public function ' . $fetchFunctionName[$scope] . '(') !== false) {
+                        $start = $lineNumber;
+                    }
+                    if ($start) {
+                        if ($lineNumber !== $start && strpos($line, 'public function') !== false) {
+                            $end = $lineNumber - 1;
+                            break;
+                        }
+                        $analyzedBlock[] = $line;
+                    }
+                }
+                $analyzedBlock = implode("\n", $analyzedBlock);
+                $foundParams = [];
+                preg_match_all('/\$options\[\'([^\']+)/i', $analyzedBlock, $foundParams);
+                $foundParams = $foundParams[1];
+                foreach ($foundParams as $k => $v) {
+                    if (in_array(strtolower($v), ['contain', 'fields', 'conditions', 'order', 'joins', 'group', 'limit', 'page', 'recursive', 'callbacks'])) {
+                        unset($foundParams[$k]);
+                    }
+                }
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $foundParams)));
+            }
+        }
+        foreach ($paramArray as $scope => $fields) {
+            echo "'" . $scope ."' => [" . PHP_EOL . "    '";
+            echo implode("'," . PHP_EOL . "    '", $fields) . "'" . PHP_EOL;
+            echo "]," . PHP_EOL;
+        }
+    }
 }

app/Console/Command/EventShell.php

@@ -91,12 +91,14 @@ public function import()
         }
 
         $isXml = $extension === 'xml';
-        $takeOwnership = $this->param('take_ownership');
-        $publish = $this->param('publish');
+        $takeOwnership = $this->params['take-ownership'];
+        $publish = $this->params['publish'];
         $results = $this->Event->addMISPExportFile($user, $content, $isXml, $takeOwnership, $publish);
 
         foreach ($results as $result) {
             if (is_numeric($result['result'])) {
+                $this->out("Event `{$result['info']}` already exists at ({$result['result']}).");
+            } else if ($result['result'] === true) {
                 $this->out("Event #{$result['id']}: {$result['info']} imported.");
             } else {
                 $this->out("Could not import event because of validation errors: " . json_encode($result['validationIssues']));

app/Console/Command/TrainingShell.php

@@ -11,7 +11,7 @@
 
 class TrainingShell extends AppShell {
 
-    public $uses = array('User', 'Organisation', 'Server', 'Authkey');
+    public $uses = array('User', 'Organisation', 'Server', 'AuthKey');
 
     private $__currentUrl = false;
     private $__currentAuthKey = false;
@@ -156,7 +156,9 @@ public function createUsersFromConfig($createdOrgs)
         $config = json_decode($rawConfig, true);
         $createdUsers = [];
         foreach ($config as $user) {
-            $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            if (!empty($user['org_uuid'])) {
+                $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            }
             $existingUser = $this->User->find('first', [
                 'recursive' => -1,
                 'conditions' => ['User.email' => $user['email']],
@@ -232,6 +234,11 @@ public function WipeAllOrgs()
         $this->Organisation->deleteAll(['Organisation.name !=' => 'ORGNAME']);
     }
 
+    public function WipeAllAuthkeys()
+    {
+        $this->AuthKey->deleteAll(['AuthKey.id !=' => 0]);
+    }
+
     private function __createOrgFromBlueprint($id)
     {
         $org = str_replace('$ID', $id, $this->__config['org_blueprint']);

app/Controller/AppController.php

@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '155';
-    public $pyMispVersion = '2.4.176';
+    public $pyMispVersion = '2.4.178';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';

app/Controller/AuthKeysController.php

@@ -279,6 +279,7 @@ private function __canEditAuthKey($key_id)
             'conditions' => [
                 'AuthKey.id' => $key_id
             ]]);
+        if(!empty($user_id)) $user_id = $user_id[0];
         return $this->__canCreateAuthKeyForUser($user_id);
     }
 }

app/Controller/Component/RestResponseComponent.php

@@ -323,6 +323,11 @@ class RestResponseComponent extends Component
                 'description' => 'Simply GET the url endpoint to view the API output of the statistics API. Additional statistics are available via the following tab-options similar to the UI: data, orgs, users, tags, attributehistogram, sightings, attackMatrix',
                 'params' => array('tab'),
                 'http_method' => 'GET'
+            ),
+            'totp_delete' => array(
+                'description' => 'Simply do a DELETE or POST request to the url',
+                'params' => array('user_id'),
+                'http_method' => 'DELETE'
             )
         ),
         'UserSetting' => array(