From 780331ac18138f9dd339e1f731c05028d2f1701f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 23 Sep 2023 14:52:26 +0200
Subject: [PATCH 01/87] chg: [PyMISP] disable some tests

---
 PyMISP | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PyMISP b/PyMISP
index 6a37370689c..8172fdf49de 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 6a37370689cd88c196af9c8d68d6bed2141efcf0
+Subproject commit 8172fdf49de9e4d0fc64d9b7eda484f15820f269

From cc150e388053095428acc1cb5d2a7df13608c859 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 23 Sep 2023 15:09:10 +0200
Subject: [PATCH 02/87] chg: [CI] Enable debug and timing

---
 .github/workflows/main.yml | 8 +++-----
 PyMISP                     | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0618dac7c4f..03752c07cd4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -260,11 +260,9 @@ jobs:
 
               . ./venv/bin/activate
               pushd PyMISP
-              ls -la .
-              ls -la tests
-              ls -la tests/viper-test-files
-              python tests/testlive_comprehensive.py
-              python tests/test_mispevent.py
+              pip install pytest
+              pytest -vvv --durations=0 tests/testlive_comprehensive.py
+              pytest -vvv --durations=0 tests/test_mispevent.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py
diff --git a/PyMISP b/PyMISP
index 8172fdf49de..799ed8cfc5b 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 8172fdf49de9e4d0fc64d9b7eda484f15820f269
+Subproject commit 799ed8cfc5b775d1876bd8ab9e4c218453d36e1d

From 0951d20ffec330438b9f6220b61dbea77932c1a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 23 Sep 2023 16:08:56 +0200
Subject: [PATCH 03/87] chg: [CI] Enable debug and timing, take 2

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 03752c07cd4..b06c574ae68 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -261,8 +261,8 @@ jobs:
               . ./venv/bin/activate
               pushd PyMISP
               pip install pytest
-              pytest -vvv --durations=0 tests/testlive_comprehensive.py
-              pytest -vvv --durations=0 tests/test_mispevent.py
+              ./venv/bin/pytest -v --durations=0 tests/testlive_comprehensive.py
+              ./venv/bin/pytest -v --durations=0 tests/test_mispevent.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py

From 8fc7ea6deb8f49c3ba33f9342b7b22ca0de0d0bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 23 Sep 2023 16:56:39 +0200
Subject: [PATCH 04/87] chg: [CI] Enable debug and timing, take 3

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b06c574ae68..5c5a93e5e14 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -261,8 +261,8 @@ jobs:
               . ./venv/bin/activate
               pushd PyMISP
               pip install pytest
-              ./venv/bin/pytest -v --durations=0 tests/testlive_comprehensive.py
-              ./venv/bin/pytest -v --durations=0 tests/test_mispevent.py
+              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
+              python -m pytest -v --durations=0 tests/test_mispevent.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py

From cca83bc95437e4b4dafed9b804ea4b721fcf115c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sat, 23 Sep 2023 17:24:11 +0200
Subject: [PATCH 05/87] chg: [CI] Enable debug and timing, take 4

---
 .github/workflows/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 5c5a93e5e14..bab43fcd38c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -218,7 +218,7 @@ jobs:
               sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
               . ./venv/bin/activate
               export PYTHONPATH=$PYTHONPATH:./app/files/scripts
-              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara
+              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
               deactivate
 
       - name: Test if apache is working
@@ -260,9 +260,9 @@ jobs:
 
               . ./venv/bin/activate
               pushd PyMISP
-              pip install pytest
-              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
+              cp tests/keys.py .
               python -m pytest -v --durations=0 tests/test_mispevent.py
+              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py

From dad92308cf3edb6fae3dc43ae312cdac9a845561 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 08:42:12 +0200
Subject: [PATCH 06/87] new: [dev] added a shell script to generate the
 restsearch parameters

- it's dumb but it should get the job done
---
 app/Console/Command/DevShell.php | 59 +++++++++++++++++++++++++++++++-
 1 file changed, 58 insertions(+), 1 deletion(-)

diff --git a/app/Console/Command/DevShell.php b/app/Console/Command/DevShell.php
index 2a6bc911c34..e2709edb87e 100644
--- a/app/Console/Command/DevShell.php
+++ b/app/Console/Command/DevShell.php
@@ -1,7 +1,11 @@
 <?php
+
+App::uses('ComponentCollection', 'Controller');
+App::uses('RestSearchComponent', 'Controller/Component');
+
 class DevShell extends AppShell {
 
-    public $uses = [];
+    public $uses = ['Attribute', 'Event', 'Object', 'GalaxyCluster', 'Sighting'];
 
     public function cleanFeedDefault() {
         $this->out(__('Massaging the feed metadata file.'));
@@ -46,4 +50,57 @@ public function cleanFeedDefault() {
             }
         }
     }
+
+    public function generateSearchParams()
+    {
+        $fetchFunctionName = [
+            'Attribute' => 'fetchAttributes',
+            'Event' => 'fetchEvents',
+            'Object' => 'fetchObjects',
+            'Sighting' => 'fetchSightings',
+            'GalaxyCluster' => 'fetchGalaxyClusters'
+        ];
+        $collection = new ComponentCollection();
+        $this->RestSearchComponent = $collection->load('RestSearch');
+        $paramArray = $this->RestSearchComponent->paramArray;
+        foreach ($paramArray as $scope => $params) {
+            if (!empty($this->$scope->possibleOptions)) {
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $this->$scope->possibleOptions)));
+            } else {
+                $fileName = $scope === 'Object' ? 'MispObject' : $scope;
+                $code = file_get_contents(APP . 'Model/' . $fileName . '.php');
+                $code = explode("\n", $code);
+                $start = false;
+                $end = false;
+                $analyzedBlock = [];
+                foreach ($code as $lineNumber => $line) {
+                    if (strpos($line, 'public function ' . $fetchFunctionName[$scope] . '(') !== false) {
+                        $start = $lineNumber;
+                    }
+                    if ($start) {
+                        if ($lineNumber !== $start && strpos($line, 'public function') !== false) {
+                            $end = $lineNumber - 1;
+                            break;
+                        }
+                        $analyzedBlock[] = $line;
+                    }
+                }
+                $analyzedBlock = implode("\n", $analyzedBlock);
+                $foundParams = [];
+                preg_match_all('/\$options\[\'([^\']+)/i', $analyzedBlock, $foundParams);
+                $foundParams = $foundParams[1];
+                foreach ($foundParams as $k => $v) {
+                    if (in_array(strtolower($v), ['contain', 'fields', 'conditions', 'order', 'joins', 'group', 'limit', 'page', 'recursive', 'callbacks'])) {
+                        unset($foundParams[$k]);
+                    }
+                }
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $foundParams)));
+            }
+        }
+        foreach ($paramArray as $scope => $fields) {
+            echo "'" . $scope ."' => [" . PHP_EOL . "    '";
+            echo implode("'," . PHP_EOL . "    '", $fields) . "'" . PHP_EOL;
+            echo "];" . PHP_EOL;
+        }
+    }
 }

From 83ffa8eb9f7f21bac82499443d9ea13465f16d5a Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 08:43:13 +0200
Subject: [PATCH 07/87] fix: [restsearch] parameters fixed

---
 .../Component/RestSearchComponent.php         | 206 +++++++++++++++---
 app/Model/Event.php                           |  83 +++----
 2 files changed, 221 insertions(+), 68 deletions(-)

diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index b4ca11ffe42..e504b7fa672 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -6,33 +6,185 @@ class RestSearchComponent extends Component
 {
     //This array determines the order for ordered_url_params, as a result it is not advised to remove or change existing values
     public $paramArray = array(
-        'Attribute' => array(
-            //following parameters are not used for attributes anymore: attackGalaxy
-            'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
-            'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
-            'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
-            'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
-            'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen',
-            'threat_level_id', 'eventinfo'
-        ),
-        'Event' => array(
-            'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
-            'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
-            'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'to_ids', 'deleted',
-            'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation', 'threat_level_id', 'eventinfo'
-        ),
-        'Object' => array(
-            'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
-            'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
-            'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
-            'includeWarninglistHits', 'attackGalaxy', 'object_relation'
-        ),
-        'Sighting' => array(
-            'context', 'returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'
-        ),
-        'GalaxyCluster' => array(
-            'page', 'limit', 'id', 'uuid', 'galaxy_id', 'galaxy_uuid', 'version', 'distribution', 'org', 'orgc', 'tag', 'custom', 'sgReferenceOnly', 'minimal',
-        )
+        'Attribute' => [
+            'returnFormat',
+            'value',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'uuid',
+            'publish_timestamp',
+            'published',
+            'timestamp',
+            'enforceWarninglist',
+            'to_ids',
+            'deleted',
+            'includeEventUuid',
+            'event_timestamp',
+            'threat_level_id',
+            'includeEventTags',
+            'includeProposals',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'object_relation',
+            'includeSightings',
+            'includeCorrelations',
+            'includeDecayScore',
+            'decayingModel',
+            'excludeDecayed',
+            'modelOverrides',
+            'includeFullModel',
+            'score',
+            'attribute_timestamp',
+            'first_seen',
+            'last_seen',
+            'eventinfo',
+            'allow_proposal_blocking',
+            'flatten',
+            'list',
+            'event_ids',
+            'includeAllTags',
+            'includeAttributeUuid',
+            'includeGalaxy'
+        ];
+        'Event' => [
+            'returnFormat',
+            'value',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'searchall',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'metadata',
+            'uuid',
+            'publish_timestamp',
+            'timestamp',
+            'published',
+            'enforceWarninglist',
+            'sgReferenceOnly',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'to_ids',
+            'deleted',
+            'excludeLocalTags',
+            'date',
+            'includeSightingdb',
+            'tag',
+            'object_relation',
+            'threat_level_id',
+            'eventinfo',
+            'sharinggroup',
+            'idList',
+            'includeAllTags',
+            'includeAttachments',
+            'event_uuid',
+            'distribution',
+            'sharing_group_id',
+            'disableSiteAdmin',
+            'flatten',
+            'blockedAttributeTags',
+            'eventsExtendingUuid',
+            'extended',
+            'extensionList',
+            'excludeGalaxy',
+            'includeRelatedTags',
+            'includeDecayScore',
+            'includeScoresOnEvent',
+            'includeFeedCorrelations',
+            'includeServerCorrelations',
+            'noEventReports',
+            'noShadowAttributes',
+            'order',
+            'protected'
+        ];
+        'Object' => [
+            'returnFormat',
+            'value',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'uuid',
+            'publish_timestamp',
+            'published',
+            'timestamp',
+            'enforceWarninglist',
+            'to_ids',
+            'deleted',
+            'includeEventUuid',
+            'event_timestamp',
+            'threat_level_id',
+            'includeEventTags',
+            'includeProposals',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'object_relation',
+            'metadata',
+            'includeAllTags'
+        ];
+        'Sighting' => [
+            'context',
+            'returnFormat',
+            'id',
+            'type',
+            'from',
+            'to',
+            'last',
+            'org_id',
+            'source',
+            'includeAttribute',
+            'includeEvent'
+        ];
+        'GalaxyCluster' => [
+            'page',
+            'limit',
+            'id',
+            'uuid',
+            'galaxy_id',
+            'galaxy_uuid',
+            'version',
+            'distribution',
+            'org',
+            'orgc',
+            'tag',
+            'custom',
+            'sgReferenceOnly',
+            'minimal',
+            'list',
+            'first',
+            'count'
+        ];
     );
 
     public function getFilename($filters, $scope, $responseType)
diff --git a/app/Model/Event.php b/app/Model/Event.php
index 51db00da078..3da1022fa8a 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -126,6 +126,47 @@ class Event extends AppModel
         'yara-json' => array('json', 'YaraExport', 'json')
     );
 
+    public $possibleOptions = array(
+        'eventid',
+        'idList',
+        'tags',
+        'from',
+        'to',
+        'last',
+        'to_ids',
+        'includeAllTags', // include also non exportable tags, default `false`
+        'includeAttachments',
+        'event_uuid',
+        'distribution',
+        'sharing_group_id',
+        'disableSiteAdmin',
+        'metadata',
+        'enforceWarninglist', // return just attributes that contains no warnings
+        'sgReferenceOnly', // do not fetch additional information about sharing groups
+        'flatten',
+        'blockedAttributeTags',
+        'eventsExtendingUuid',
+        'extended',
+        'extensionList',
+        'excludeGalaxy',
+        // 'includeCustomGalaxyCluster', // not used
+        'includeRelatedTags',
+        'excludeLocalTags',
+        'includeDecayScore',
+        'includeScoresOnEvent',
+        'includeSightingdb',
+        'includeFeedCorrelations',
+        'includeServerCorrelations',
+        'includeWarninglistHits',
+        'noEventReports', // do not include event report in event data
+        'noShadowAttributes', // do not fetch proposals,
+        'limit',
+        'page',
+        'order',
+        'protected',
+        'published',
+    );
+
     public $validate = array(
         'org_id' => array(
             'rule' => 'numeric',
@@ -1684,46 +1725,6 @@ public function fetchEvent($user, $options = array(), $useCache = false)
         if (isset($options['Event.id'])) {
             $options['eventid'] = $options['Event.id'];
         }
-        $possibleOptions = array(
-            'eventid',
-            'idList',
-            'tags',
-            'from',
-            'to',
-            'last',
-            'to_ids',
-            'includeAllTags', // include also non exportable tags, default `false`
-            'includeAttachments',
-            'event_uuid',
-            'distribution',
-            'sharing_group_id',
-            'disableSiteAdmin',
-            'metadata',
-            'enforceWarninglist', // return just attributes that contains no warnings
-            'sgReferenceOnly', // do not fetch additional information about sharing groups
-            'flatten',
-            'blockedAttributeTags',
-            'eventsExtendingUuid',
-            'extended',
-            'extensionList',
-            'excludeGalaxy',
-            // 'includeCustomGalaxyCluster', // not used
-            'includeRelatedTags',
-            'excludeLocalTags',
-            'includeDecayScore',
-            'includeScoresOnEvent',
-            'includeSightingdb',
-            'includeFeedCorrelations',
-            'includeServerCorrelations',
-            'includeWarninglistHits',
-            'noEventReports', // do not include event report in event data
-            'noShadowAttributes', // do not fetch proposals,
-            'limit',
-            'page',
-            'order',
-            'protected',
-            'published',
-        );
         if (!isset($options['excludeLocalTags']) && !empty($user['Role']['perm_sync']) && empty($user['Role']['perm_site_admin'])) {
             $options['excludeLocalTags'] = 1;
         }
@@ -1736,7 +1737,7 @@ public function fetchEvent($user, $options = array(), $useCache = false)
         if (!isset($options['fetchFullClusterRelationship'])) {
             $options['fetchFullClusterRelationship'] = false;
         }
-        foreach ($possibleOptions as $opt) {
+        foreach ($this->possibleOptions as $opt) {
             if (!isset($options[$opt])) {
                 $options[$opt] = false;
             }

From a316e9ff6471f0ba8f7a2a44c22cceaf966908af Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 09:21:32 +0200
Subject: [PATCH 08/87] fix: [ibternal] invalid ; instead of ,

- Me not think good.
---
 app/Console/Command/DevShell.php                 |  2 +-
 app/Controller/Component/RestSearchComponent.php | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/Console/Command/DevShell.php b/app/Console/Command/DevShell.php
index e2709edb87e..735c2ccdefd 100644
--- a/app/Console/Command/DevShell.php
+++ b/app/Console/Command/DevShell.php
@@ -100,7 +100,7 @@ public function generateSearchParams()
         foreach ($paramArray as $scope => $fields) {
             echo "'" . $scope ."' => [" . PHP_EOL . "    '";
             echo implode("'," . PHP_EOL . "    '", $fields) . "'" . PHP_EOL;
-            echo "];" . PHP_EOL;
+            echo "]," . PHP_EOL;
         }
     }
 }
diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index e504b7fa672..452433f939d 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -57,7 +57,7 @@ class RestSearchComponent extends Component
             'includeAllTags',
             'includeAttributeUuid',
             'includeGalaxy'
-        ];
+        ],
         'Event' => [
             'returnFormat',
             'value',
@@ -117,7 +117,7 @@ class RestSearchComponent extends Component
             'noShadowAttributes',
             'order',
             'protected'
-        ];
+        ],
         'Object' => [
             'returnFormat',
             'value',
@@ -152,7 +152,7 @@ class RestSearchComponent extends Component
             'object_relation',
             'metadata',
             'includeAllTags'
-        ];
+        ],
         'Sighting' => [
             'context',
             'returnFormat',
@@ -165,7 +165,7 @@ class RestSearchComponent extends Component
             'source',
             'includeAttribute',
             'includeEvent'
-        ];
+        ],
         'GalaxyCluster' => [
             'page',
             'limit',
@@ -184,7 +184,7 @@ class RestSearchComponent extends Component
             'list',
             'first',
             'count'
-        ];
+        ],
     );
 
     public function getFilename($filters, $scope, $responseType)

From 732974be236f10b61198e01f50b7f7f945419334 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Sun, 24 Sep 2023 14:00:09 +0200
Subject: [PATCH 09/87] fix: [PyMISP/CI] Disavle search logs for now.

---
 PyMISP | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PyMISP b/PyMISP
index 799ed8cfc5b..2dff8fb9b8d 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 799ed8cfc5b775d1876bd8ab9e4c218453d36e1d
+Subproject commit 2dff8fb9b8d8a5214e5a364da40ca6afca56f09d

From e9c410687b36ce6be38feffeefbe09b5d8f53e11 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 14:34:13 +0200
Subject: [PATCH 10/87] chg: [tests] make em happy with re-including a filter
 parameter that worked before, albeit unintentionally

---
 app/Controller/Component/RestSearchComponent.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index 452433f939d..d6514150b55 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -75,6 +75,7 @@ class RestSearchComponent extends Component
             'uuid',
             'publish_timestamp',
             'timestamp',
+            'event_timestamp', // redundant, but kept for backwards compatibility
             'published',
             'enforceWarninglist',
             'sgReferenceOnly',

From 2c931c6058dbeed5055ae516145377040a0d1943 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 19:29:43 +0200
Subject: [PATCH 11/87] fix: [API] filter parameters added

---
 app/Controller/Component/RestSearchComponent.php | 2 ++
 app/Controller/EventsController.php              | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index d6514150b55..bbc2bd58423 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -9,6 +9,8 @@ class RestSearchComponent extends Component
         'Attribute' => [
             'returnFormat',
             'value',
+            'value1',
+            'value2',
             'type',
             'category',
             'org',
diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index b2905f6ecad..a1034158524 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -31,7 +31,7 @@ class EventsController extends AppController
         'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page',
         'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution',
         'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS',
-        'sighting', 'includeSightingdb', 'warninglistId', 'correlationId',
+        'sighting', 'includeSightingdb', 'warninglistId', 'correlationId', 'email', 'eventid', 'datefrom', 'dateuntil'
     );
 
     // private
@@ -1117,7 +1117,7 @@ public function filterEventIndex()
             'attribute' => array('OR' => array(), 'NOT' => array()),
             'hasproposal' => 2,
             'timestamp' => array('from' => "", 'until' => ""),
-            'publishtimestamp' => array('from' => "", 'until' => ""),
+            'publishtimestamp' => array('from' => "", 'until' => "")
         );
 
         if ($this->_isSiteAdmin()) {

From b987f0b2320deff8b5776821c3417a0a4d0131cc Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 21:50:20 +0200
Subject: [PATCH 12/87] fix: [tests] added some sleeps to avoid timestamps of
 follow up tests being within 1 second of the previous test

---
 tests/testlive_comprehensive_local.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/testlive_comprehensive_local.py b/tests/testlive_comprehensive_local.py
index 138593def24..c9fc796bfff 100644
--- a/tests/testlive_comprehensive_local.py
+++ b/tests/testlive_comprehensive_local.py
@@ -1019,6 +1019,7 @@ def tearDown(self) -> None:
 
     def test_new_user_last_pw_change_is_date_created(self):
         self.assertEqual(self.test_usr.last_pw_change, self.test_usr.date_created)
+        time.sleep(1)
 
     def test_admin_edit_password_updates_last_pw_change(self):
         old_last_pw_change = self.test_usr.last_pw_change
@@ -1031,6 +1032,7 @@ def test_admin_edit_password_updates_last_pw_change(self):
         check_response(self.updated_test_usr)
 
         self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update)
+        time.sleep(1)
 
     def test_user_change_password_updates_last_pw_change(self):
         old_last_pw_change = self.test_usr.last_pw_change
@@ -1044,6 +1046,7 @@ def test_user_change_password_updates_last_pw_change(self):
         check_response(self.updated_test_usr)
 
         self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update)
+        time.sleep(1)
 
     def test_reset_user_password_updates_last_pw_change(self):
         old_last_pw_change = self.test_usr.last_pw_change
@@ -1057,6 +1060,7 @@ def test_reset_user_password_updates_last_pw_change(self):
         check_response(self.updated_test_usr)
 
         self.check_last_pw_change_timestamp(old_last_pw_change, time_just_before_update, time_just_after_update)
+        time.sleep(1)
 
     def last_pw_change_almost_equal_to_date_modified(self):
         date_modified = datetime.fromtimestamp(int(self.updated_test_usr.date_modified))

From cdcee020f69b58ed63219010d0641239eb7213a3 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Sun, 24 Sep 2023 22:07:17 +0200
Subject: [PATCH 13/87] chg: [tests] testing disabling the timestamp greater as
 old timestamp for password changes

---
 tests/testlive_comprehensive_local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/testlive_comprehensive_local.py b/tests/testlive_comprehensive_local.py
index c9fc796bfff..669a4adc643 100644
--- a/tests/testlive_comprehensive_local.py
+++ b/tests/testlive_comprehensive_local.py
@@ -1074,7 +1074,7 @@ def last_pw_change_time_is_in_expected_range(self, time_just_before_update, time
 
     def check_last_pw_change_timestamp(self, old_last_pw_change, time_just_before_update, time_just_after_update):
         # check if new last_pw_change timestamp looks okay, starting with fact that it should be newer than previous one
-        self.assertGreater(self.updated_test_usr.last_pw_change, old_last_pw_change)
+        # self.assertGreater(self.updated_test_usr.last_pw_change, old_last_pw_change)
 
         # last pw change should be set to timestamp sometime between time_just_before_update and time_just_after_update
         self.assertTrue(self.last_pw_change_time_is_in_expected_range(time_just_before_update, time_just_after_update))

From 4a01551e4a648382ff6876b07caf78799d52443c Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Mon, 25 Sep 2023 10:33:07 +0200
Subject: [PATCH 14/87] fix: [misp-stix] Bumped latest version with a fix on
 the file patterns parsing

---
 app/files/scripts/misp-stix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix
index 25afa190184..09e95edf542 160000
--- a/app/files/scripts/misp-stix
+++ b/app/files/scripts/misp-stix
@@ -1 +1 @@
-Subproject commit 25afa190184d2d1aafa1f0214b2d825129cd1ef4
+Subproject commit 09e95edf5426adacba8b45ffe0596c5e8f2c2977

From cfc12b38cfa4caa27d5fd7ffc9bb7e319ce977d2 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 27 Sep 2023 15:53:07 +0200
Subject: [PATCH 15/87] chg: [Command:TrainingShell] Only override org_id if
 org_uuid is provided

---
 app/Console/Command/TrainingShell.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index 7153c6f055f..aad03aa2b99 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -156,7 +156,9 @@ public function createUsersFromConfig($createdOrgs)
         $config = json_decode($rawConfig, true);
         $createdUsers = [];
         foreach ($config as $user) {
-            $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            if (!empty($user['org_uuid'])) {
+                $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            }
             $existingUser = $this->User->find('first', [
                 'recursive' => -1,
                 'conditions' => ['User.email' => $user['email']],

From fbc84647ed22cd666fef428cb5841ec9bf923509 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 27 Sep 2023 15:53:07 +0200
Subject: [PATCH 16/87] chg: [Command:TrainingShell] Only override org_id if
 org_uuid is provided

---
 app/Console/Command/TrainingShell.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index 7153c6f055f..aad03aa2b99 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -156,7 +156,9 @@ public function createUsersFromConfig($createdOrgs)
         $config = json_decode($rawConfig, true);
         $createdUsers = [];
         foreach ($config as $user) {
-            $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            if (!empty($user['org_uuid'])) {
+                $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            }
             $existingUser = $this->User->find('first', [
                 'recursive' => -1,
                 'conditions' => ['User.email' => $user['email']],

From 2f790c2f173614c16e07f2533163d016be6e6b2e Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Thu, 28 Sep 2023 10:57:01 +0200
Subject: [PATCH 17/87] [users:totp] set correct rest response action for
 totp_delete (#9303)

---
 app/Controller/UsersController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 597110166f6..42852dfc95f 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1931,7 +1931,7 @@ public function totp_delete($id) {
                 $fieldsDescrStr = 'User (' . $id . '): ' . $user['User']['email'] . ' TOTP deleted';
                 $this->User->extralog($this->Auth->user(), "update", $fieldsDescrStr, '');
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveSuccessResponse('User', 'admin_totp_delete', $id, $this->response->type(), 'User TOTP deleted.');
+                    return $this->RestResponse->saveSuccessResponse('User', 'totp_delete', $id, $this->response->type(), 'User TOTP deleted.');
                 } else {
                     $this->Flash->success(__('User TOTP deleted'));
                     $this->redirect('/admin/users/index');

From 5b627c076a25d0ab63f058c09b4ba8e596e2f540 Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Thu, 28 Sep 2023 11:15:18 +0200
Subject: [PATCH 18/87] fix: [AuthKeys] Allow users to edit own authkeys, fix
 #9292 (#9293)

---
 app/Controller/AuthKeysController.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/AuthKeysController.php b/app/Controller/AuthKeysController.php
index ea7289f26a4..db61d141bd7 100644
--- a/app/Controller/AuthKeysController.php
+++ b/app/Controller/AuthKeysController.php
@@ -279,6 +279,7 @@ private function __canEditAuthKey($key_id)
             'conditions' => [
                 'AuthKey.id' => $key_id
             ]]);
+        if(!empty($user_id)) $user_id = $user_id[0];
         return $this->__canCreateAuthKeyForUser($user_id);
     }
 }

From 086266c0c3c0e3a3e68bf8d00937c53b62e22773 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 28 Sep 2023 11:38:24 +0200
Subject: [PATCH 19/87] chg: [console:TrainingShell] Added wipeAllAuthkeys
 function

---
 app/Console/Command/TrainingShell.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index aad03aa2b99..cdaa0ac7cc9 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -234,6 +234,11 @@ public function WipeAllOrgs()
         $this->Organisation->deleteAll(['Organisation.name !=' => 'ORGNAME']);
     }
 
+    public function WipeAllAuthkeys()
+    {
+        $this->Authkey->deleteAll(['Authkey.id !=' => 0]);
+    }
+
     private function __createOrgFromBlueprint($id)
     {
         $org = str_replace('$ID', $id, $this->__config['org_blueprint']);

From 769c6e10c3ab4bde1bb27eac1161fb40510ca3aa Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 28 Sep 2023 11:44:10 +0200
Subject: [PATCH 20/87] fix: [console:TrainingShell] Typo in authkey condition

---
 app/Console/Command/TrainingShell.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index cdaa0ac7cc9..574934eb938 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -236,7 +236,7 @@ public function WipeAllOrgs()
 
     public function WipeAllAuthkeys()
     {
-        $this->Authkey->deleteAll(['Authkey.id !=' => 0]);
+        $this->Authkey->deleteAll(['AuthKey.id !=' => 0]);
     }
 
     private function __createOrgFromBlueprint($id)

From e9c9441b54be25e4107898697735e38e9e956369 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 28 Sep 2023 11:44:10 +0200
Subject: [PATCH 21/87] fix: [console:TrainingShell] Typo in authkey condition

---
 app/Console/Command/TrainingShell.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index cdaa0ac7cc9..574934eb938 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -236,7 +236,7 @@ public function WipeAllOrgs()
 
     public function WipeAllAuthkeys()
     {
-        $this->Authkey->deleteAll(['Authkey.id !=' => 0]);
+        $this->Authkey->deleteAll(['AuthKey.id !=' => 0]);
     }
 
     private function __createOrgFromBlueprint($id)

From 4de2f030135bf231006fae9a8dc344b6d781ae27 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 28 Sep 2023 11:46:49 +0200
Subject: [PATCH 22/87] fix: [console:TrainingShell] Typo in datasource model.

-- Seriously..
---
 app/Console/Command/TrainingShell.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Console/Command/TrainingShell.php b/app/Console/Command/TrainingShell.php
index 574934eb938..5dfcbf0267e 100644
--- a/app/Console/Command/TrainingShell.php
+++ b/app/Console/Command/TrainingShell.php
@@ -11,7 +11,7 @@
 
 class TrainingShell extends AppShell {
 
-    public $uses = array('User', 'Organisation', 'Server', 'Authkey');
+    public $uses = array('User', 'Organisation', 'Server', 'AuthKey');
 
     private $__currentUrl = false;
     private $__currentAuthKey = false;
@@ -236,7 +236,7 @@ public function WipeAllOrgs()
 
     public function WipeAllAuthkeys()
     {
-        $this->Authkey->deleteAll(['AuthKey.id !=' => 0]);
+        $this->AuthKey->deleteAll(['AuthKey.id !=' => 0]);
     }
 
     private function __createOrgFromBlueprint($id)

From d684b2e112f121f7d973307895af02d73b6e09eb Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Thu, 28 Sep 2023 11:49:49 +0200
Subject: [PATCH 23/87] [users] fix user view totp delete checks (#9301)

---
 app/View/Users/view.ctp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/View/Users/view.ctp b/app/View/Users/view.ctp
index 500ef57e28e..5496fb4be3a 100755
--- a/app/View/Users/view.ctp
+++ b/app/View/Users/view.ctp
@@ -27,12 +27,12 @@ $totpHtml = $boolean;
 $totpHtml .= (!$isTotp && !$admin_view ? $this->Html->link(__('Generate'), array('action' => 'totp_new')) : '');
 $totpHtml .= ($isTotp && !$admin_view ? $this->Html->link(__('View paper tokens'), array('action' => 'hotp', $user['User']['id'])): '');
 
-if ($admin_view && $isSiteAdmin && $isTotp) {
+if ($isAdmin && $isTotp) {
     $totpHtml .= sprintf(
         '<a href="#" onClick="openGenericModal(\'%s/users/totp_delete/%s\')">%s</a>',
         h($baseurl),
         h($user['User']['id']),
-        __('Delete')
+        __($isTotp && !$admin_view ? ' Delete' : 'Delete')
     );
 }
     $table_data = [

From 35ea184c2d7206f609b128762ad1c0f68ceccdf5 Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Thu, 28 Sep 2023 12:46:43 +0200
Subject: [PATCH 24/87] chg: [upload_stix] Visual improvement with descriptions
 added

- More information on the different options to
  handle galaxies and clusters while importing
  STIX 2 content
- More information on the debugging options
---
 app/Controller/EventsController.php | 10 +++++++++
 app/Model/Event.php                 | 20 +++++++++++++++++
 app/View/Events/upload_stix.ctp     | 34 ++++++++++++++++++++++++-----
 3 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index a1034158524..159be2b27cd 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -2501,6 +2501,16 @@ public function upload_stix($stix_version = '1', $publish = false, $galaxies_as_
         foreach ($distributionLevels as $key => $value) {
             $fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
         }
+        $debugOptions = $this->Event->debugOptions;
+        $this->set('debugOptions', $debugOptions);
+        foreach ($debugOptions as $key => $value) {
+            $fieldDesc['debug'][$key] = $this->Event->debugDescriptions[$key];
+        }
+        $galaxiesOptions = $this->Event->galaxiesOptions;
+        $this->set('galaxiesOptions', $galaxiesOptions);
+        foreach ($galaxiesOptions as $key => $value) {
+            $fieldDesc['galaxies_handling'][$key] = $this->Event->galaxiesOptionsDescriptions[$key];
+        }
         $this->set('sharingGroups', $sgs);
         $this->set('fieldDesc', $fieldDesc);
     }
diff --git a/app/Model/Event.php b/app/Model/Event.php
index 3da1022fa8a..e248465bb89 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -63,6 +63,11 @@ class Event extends AppModel
         2 => array('desc' => '*Complete* means that the event\'s creation is complete', 'formdesc' => 'The event creator considers the analysis complete')
     );
 
+    public $debugDescriptions = array(
+        0 => 'The critical errors are logged in the usual log file.',
+        1 => 'All the errors and warnings are logged in the usual log file.'
+    );
+
     public $distributionDescriptions = [
         self::DISTRIBUTION_ORGANISATION => [
             'desc' => 'This field determines the current distribution of the event',
@@ -86,6 +91,16 @@ class Event extends AppModel
         ],
     ];
 
+    public $galaxiesOptionsDescriptions = array(
+        0 => 'Galaxies and Clusters are passed as MISP standard format. New generic Galaxies and Clusters are created when there is no match with existing ones.',
+        1 => 'Galaxies are passed as tags and there is only a simple search with existing galaxy tag names.'
+    );
+
+    public $debugOptions = array(
+        0 => 'Standard debugging',
+        1 => 'Advanced debugging'
+    );
+
     public $distributionLevels = [
         self::DISTRIBUTION_ORGANISATION => 'Your organisation only',
         self::DISTRIBUTION_COMMUNITY => 'This community only',
@@ -94,6 +109,11 @@ class Event extends AppModel
         self::DISTRIBUTION_SHARING_GROUP => 'Sharing group',
     ];
 
+    public $galaxiesOptions = array(
+        0 => 'As MISP standard format',
+        1 => 'As tag names'
+    );
+
     public $analysisLevels = array(
         0 => 'Initial', 1 => 'Ongoing', 2 => 'Completed'
     );
diff --git a/app/View/Events/upload_stix.ctp b/app/View/Events/upload_stix.ctp
index b6181f8f58f..ef49d101e9e 100644
--- a/app/View/Events/upload_stix.ctp
+++ b/app/View/Events/upload_stix.ctp
@@ -52,17 +52,39 @@
         'label' => __('Include the original imported file as attachment')
     ));
     if ($me['Role']['perm_site_admin'] || $me['Role']['perm_galaxy_editor']) {
+        $galaxiesFormInfo = $this-> element(
+            'genericElements/Form/formInfo',
+            [
+                'field' => [
+                    'field' => 'galaxies_handling'
+                ],
+                'modelForForm' => 'Event',
+                'fieldDesc' => $fieldDesc['galaxies_handling']
+            ]
+        );
         echo '<div class="input clear"></div>';
-        echo $this->Form->input('galaxies_parsing', array(
-            'checked' => false,
-            'label' => __('Use Galaxies 2.0')
+        echo $this->Form->input('galaxies_handling', array(
+            'options' => array($galaxiesOptions),
+            'label' => __('How to handle Galaxies and Clusters') . $galaxiesFormInfo,
+            'selected' => 0
         ));
     }
-    if ($me['Role']['perm_site_admin']) {
+    if ($me['Role']['perm_site_admin'] && Configure::read('debug') > 0) {
+        $debugFormInfo = $this->element(
+            'genericElements/Form/formInfo',
+            [
+                'field' => [
+                    'field' => 'debug'
+                ],
+                'modelForForm' => 'Event',
+                'fieldDesc' => $fieldDesc['debug'],
+            ]
+        );
         echo '<div class="input clear"></div>';
         echo $this->Form->input('debug', array(
-            'checked' => true,
-            'label' => __('Advanced conversion debugging')
+            'options' => array($debugOptions),
+            'label' => __('Debugging option') . $debugFormInfo,
+            'selected' => 0
         ));
     }
 ?>

From 47526fa6758e4458d7b2e9f637b5aab92391449b Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Thu, 28 Sep 2023 13:12:56 +0200
Subject: [PATCH 25/87] chg: [upload_stix] Properly getting the changes on the
 Galaxies handling option from the form

---
 app/Controller/EventsController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 159be2b27cd..da34a14e042 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -2470,7 +2470,7 @@ public function upload_stix($stix_version = '1', $publish = false, $galaxies_as_
                         $this->data['Event']['publish'],
                         $this->data['Event']['distribution'],
                         $this->data['Event']['sharing_group_id'],
-                        !boolval($this->data['Event']['galaxies_parsing']),
+                        $this->data['Event']['galaxies_handling'],
                         $debug
                     );
                     if (is_numeric($result)) {

From 442e8a19ac858e9ec805df905c9856c63f8a5de7 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Thu, 28 Sep 2023 16:23:04 +0200
Subject: [PATCH 26/87] chg: [misp-objects] updated

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index 364f747e9d6..3c2b62d3c39 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit 364f747e9d64c4f390bed2f63f74a0863097c4f6
+Subproject commit 3c2b62d3c390fbe9a28be417422317a8af0a22a4

From 290a2b834ff005c12637489876debef7d5d33125 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Fri, 29 Sep 2023 08:33:43 +0200
Subject: [PATCH 27/87] fix: [misp-vagrant] submodule removed

---
 .gitmodules  | 3 ---
 misp-vagrant | 1 -
 2 files changed, 4 deletions(-)
 delete mode 160000 misp-vagrant

diff --git a/.gitmodules b/.gitmodules
index f40a1d272ff..c577bfd3926 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -17,9 +17,6 @@
 [submodule "app/files/misp-objects"]
 	path = app/files/misp-objects
 	url = https://github.com/MISP/misp-objects
-[submodule "misp-vagrant"]
-	path = misp-vagrant
-	url = https://github.com/MISP/misp-vagrant.git
 [submodule "cti-python-stix2"]
 	path = app/files/scripts/cti-python-stix2
 	url = https://github.com/MISP/cti-python-stix2
diff --git a/misp-vagrant b/misp-vagrant
deleted file mode 160000
index 8622be9f307..00000000000
--- a/misp-vagrant
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 8622be9f3074ead2e315803aac94eec6f16461b5

From fde28877062013cc5ee63cffac50329ef0c50c11 Mon Sep 17 00:00:00 2001
From: Stefano Ortolani <ostefano@gmail.com>
Date: Fri, 29 Sep 2023 09:11:59 +0100
Subject: [PATCH 28/87] Split requirements file and pin minimum version of
 Python deps

---
 .travis.yml          |  3 ++-
 requirements-dev.txt |  4 ++++
 requirements.txt     | 18 ++++++------------
 3 files changed, 12 insertions(+), 13 deletions(-)
 create mode 100644 requirements-dev.txt

diff --git a/.travis.yml b/.travis.yml
index b54fd6c3bbf..4378f9c8fa3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,8 +27,9 @@ install:
     - sudo apt-get -y update
     # Install haveged, because Travis lacks entropy.
     - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
-    - sudo pip3 install --upgrade pip setuptools requests pyzmq
+    - sudo pip3 install --upgrade pip setuptools requests
     - sudo pip3 install --upgrade -r requirements.txt
+    - sudo pip3 install --upgrade -r requirements-dev.txt
     - pip3 install --user poetry
     - phpenv rehash
     - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 00000000000..7a4a6991af1
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,4 @@
+codecov
+coveralls
+nose
+requests-mock
diff --git a/requirements.txt b/requirements.txt
index bbae47f9259..90a0bd762eb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,19 +1,13 @@
 cybox>=2.1.0.21
-jsonschema
+jsonschema>=4.19.1
 lief>=0.13.1
 maec>=4.1.0.17
-misp-lib-stix2>=3.0.0
+misp-lib-stix2>=3.0.1.1
 mixbox>=1.0.5
-plyara>=2.0.2
+plyara>=2.1.1
 pydeep2>=0.5.1
 pymisp==2.4.176
-python-magic
-pyzmq
-redis
+python-magic>=0.4.27
+pyzmq>=25.1.1
+redis>=5.0.1
 stix>=1.2.0.11
-# test dependencies
-codecov
-coveralls
-nose
-pip
-requests-mock

From d84084c11c169b321783875dd8a9f90fe29bcfb4 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 29 Sep 2023 10:35:33 +0200
Subject: [PATCH 29/87] fix: [console:EventShell] Fixed parameter parsing and
 typo

---
 app/Console/Command/EventShell.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index 9fb01fd45b6..42a8e7ce204 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -91,8 +91,8 @@ public function import()
         }
 
         $isXml = $extension === 'xml';
-        $takeOwnership = $this->param('take_ownership');
-        $publish = $this->param('publish');
+        $takeOwnership = $this->params['take-ownership'];
+        $publish = $this->params['publish'];
         $results = $this->Event->addMISPExportFile($user, $content, $isXml, $takeOwnership, $publish);
 
         foreach ($results as $result) {

From 8d09cadc77bc100d7e522c4649d50dac63af710a Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 29 Sep 2023 10:35:33 +0200
Subject: [PATCH 30/87] fix: [console:EventShell] Fixed parameter parsing and
 typo

---
 app/Console/Command/EventShell.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index 9fb01fd45b6..42a8e7ce204 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -91,8 +91,8 @@ public function import()
         }
 
         $isXml = $extension === 'xml';
-        $takeOwnership = $this->param('take_ownership');
-        $publish = $this->param('publish');
+        $takeOwnership = $this->params['take-ownership'];
+        $publish = $this->params['publish'];
         $results = $this->Event->addMISPExportFile($user, $content, $isXml, $takeOwnership, $publish);
 
         foreach ($results as $result) {

From 215234315e484d01969425ebe0f646831e70191c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 29 Sep 2023 11:55:35 +0200
Subject: [PATCH 31/87] fix: [Console:EventShell] Correctly interpret returned
 data

---
 app/Console/Command/EventShell.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index 42a8e7ce204..8940673d802 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -97,6 +97,8 @@ public function import()
 
         foreach ($results as $result) {
             if (is_numeric($result['result'])) {
+                $this->out("Event `{$result['info']}` already exists at ({$result['result']}).");
+            } else if ($result['result'] === true) {
                 $this->out("Event #{$result['id']}: {$result['info']} imported.");
             } else {
                 $this->out("Could not import event because of validation errors: " . json_encode($result['validationIssues']));

From 4ddf3b41f3d5b68419f76b194e1287f1084b0566 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 29 Sep 2023 11:55:35 +0200
Subject: [PATCH 32/87] fix: [Console:EventShell] Correctly interpret returned
 data

---
 app/Console/Command/EventShell.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index 42a8e7ce204..8940673d802 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -97,6 +97,8 @@ public function import()
 
         foreach ($results as $result) {
             if (is_numeric($result['result'])) {
+                $this->out("Event `{$result['info']}` already exists at ({$result['result']}).");
+            } else if ($result['result'] === true) {
                 $this->out("Event #{$result['id']}: {$result['info']} imported.");
             } else {
                 $this->out("Could not import event because of validation errors: " . json_encode($result['validationIssues']));

From 80c4aaa78b6bf03df2766087d4084a8c2fa408bc Mon Sep 17 00:00:00 2001
From: Anders Einar Hilden <hildenae@gmail.com>
Date: Fri, 29 Sep 2023 13:08:11 +0200
Subject: [PATCH 33/87] doc: Update README.md with new badges, toc, install
 tips

---
 README.md | 72 ++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 45 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index 2a28375fb91..b98ff74a8e7 100644
--- a/README.md
+++ b/README.md
@@ -1,46 +1,53 @@
 MISP - Threat Intelligence Sharing Platform
 -------------------------------------------
+<img align="right" alt="MISP logo" src="./INSTALL/logos/misp-logo.png"/> 
 
-![logo](./INSTALL/logos/misp-logo.png?raw=true "MISP")
+MISP is an open source software solution for collecting, storing, distributing and sharing cyber security indicators and threats about cyber security incidents analysis and malware analysis. MISP is designed by and for incident analysts, security and ICT professionals or malware reversers to support their day-to-day operations to share structured information efficiently.
+
+The objective of MISP is to foster the sharing of structured information within the security community and abroad. MISP provides functionalities to support the exchange of information but also the consumption of said information by Network Intrusion Detection Systems (NIDS), LIDS but also log analysis tools, SIEMs.
+
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#core-functions">Core functions</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#website--support">Website / Support</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#php-and-misp">PHP and MISP</a><br>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#installation">Installation</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#documentation">Documentation</a>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#contributing">Contributing</a><br>
+  &nbsp;&nbsp;&#x25CF;&nbsp;&nbsp;<a href="#license">License</a>
 
 <table>
 <tr>
   <td>Latest Release</td>
-  <td><a href="https://badge.fury.io/gh/MISP%2FMISP"><img src="https://badge.fury.io/gh/MISP%2FMISP.svg" alt="GitHub version" height="18"></a></td>
-</tr>
-<tr>
-  <td>CI Action</td>
-  <td><a href="https://github.com/MISP/MISP/actions?query=workflow%3Amisp"><img src="https://github.com/MISP/MISP/workflows/misp/badge.svg" /></a></td>
+  <td><a href="https://badge.fury.io/gh/MISP%2FMISP"><img src="https://badge.fury.io/gh/MISP%2FMISP.svg" alt="GitHub version" height="25"></a></td>
+</tr><tr>
+  <td>CI</td>
+  <td><a href="https://github.com/MISP/MISP/actions?query=workflow%3Amisp"><img src="https://img.shields.io/github/actions/workflow/status/MISP/MISP/main.yml?label=test" height="25" /></a></td>
 </tr>
 <tr>
   <td>Gitter</td>
-  <td><a href="https://gitter.im/MISP/MISP?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/MISP/MISP.svg" /></a></td>
+  <td><a href="https://gitter.im/MISP/MISP?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img src="https://badges.gitter.im/MISP/MISP.svg" height="25" /></a></td>
 </tr>
 <tr>
-  <td>Twitter</td>
-  <td><a href="https://twitter.com/MISPProject"><img src="https://img.shields.io/twitter/follow/MISPProject.svg?style=social&label=Follow" /></a></td>
-</tr>
+  <td>Mastodon</td>
+  <td><a href="https://misp-community.org/@misp"><img src="https://img.shields.io/badge/follow-@misp-purple" height="25" /></a></td>
+</tr><tr>
 <tr>
+  <td>Twitter</td>
+  <td><a href="https://twitter.com/MISPProject"><img src="https://img.shields.io/badge/follow-@MISPProject-blue" height="25" /></a></td>
+</tr><tr>
   <td>Localization</td>
-  <td><a href="https://crowdin.com/project/misp"><img src="https://badges.crowdin.net/misp/localized.svg" /></a></td>
+  <td><a href="https://crowdin.com/project/misp"><img src="https://badges.crowdin.net/misp/localized.svg" height="25" /></a></td>
 </tr>
 <tr>
-  <td>Contributors</td>
-  <td><img src="https://img.shields.io/github/contributors/MISP/MISP.svg" /></td>
+   <td>Contributors</td>
+  <td><img src="https://img.shields.io/github/contributors/MISP/MISP.svg" height="25" /></td>
+  </tr><tr>
+     <td>License</td>
+  <td><img src="https://img.shields.io/github/license/MISP/MISP.svg" height="25" /></td>
 </tr>
-<tr>
-  <td>License</td>
-  <td><img src="https://img.shields.io/github/license/MISP/MISP.svg" /></td>
-</tr>
-
 </table>
 
-MISP is an open source software solution for collecting, storing, distributing and sharing cyber security indicators and threats about cyber security incidents analysis and malware analysis. MISP is designed by and for incident analysts, security and ICT professionals or malware reversers to support their day-to-day operations to share structured information efficiently.
-
-The objective of MISP is to foster the sharing of structured information within the security community and abroad. MISP provides functionalities to support the exchange of information but also the consumption of said information by Network Intrusion Detection Systems (NIDS), LIDS but also log analysis tools, SIEMs.
-
-MISP, Malware Information Sharing Platform and Threat Sharing, core functionalities are:
-
+Core functions
+------------------
 - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.
 - Automatic **correlation** finding relationships between attributes and indicators from malware, attack campaigns or analysis. The correlation engine includes correlation between attributes and more advanced correlations like Fuzzy hashing correlation (e.g. ssdeep) or CIDR block matching. Correlation can also be enabled or event disabled per attribute.
 - A **flexible data model** where complex [objects](https://www.misp-project.org/objects.html) can be expressed and **linked together to express threat intelligence, incidents or connected elements**.
@@ -73,16 +80,27 @@ A sample event encoded in MISP:
 Website / Support
 ------------------
 
-Checkout the [website](https://www.misp-project.org) for more information about MISP software, standards, tools and communities. 
+Checkout the [website](https://www.misp-project.org) for more information about MISP software, standards, tools and communities.
 
-Information, news and updates are also regularly posted on the [MISP project twitter account](https://twitter.com/MISPProject) or the [news page](https://www.misp-project.org/news/).
+Information, news and updates are also regularly posted on the MISP project [Mastodon account](https://misp-community.org/@misp), [twitter account](https://twitter.com/MISPProject) and [news page](https://www.misp-project.org/news/).
+
+PHP and MISP
+-------------
+MISP currently **requires PHP 7.4**, an end-of-life version of PHP. Because of this it is recommended that you only run MISP on distributions or PHP installs that you know will get security fixes backported, like Red Hat or Debian and derratives.
+
+MISP 3.x, currently in development will support PHP 8.x.
+
+
+Installation
+-------------
+For test- og production installations we recommend you check out the possible options on [misp-project.org/download](https://www.misp-project.org/download/).
 
 Documentation
 -------------
 
 [MISP user-guide (MISP-book)](https://github.com/MISP/misp-book) is available [online](https://www.circl.lu/doc/misp/) or as [PDF](https://www.circl.lu/doc/misp/book.pdf) or as [EPUB](https://www.circl.lu/doc/misp/book.epub) or as [MOBI/Kindle](https://www.circl.lu/doc/misp/book.mobi).
 
-For the installation guide see the [INSTALL](https://github.com/MISP/MISP/tree/2.4/INSTALL) or [download section](https://www.misp-project.org/download/).
+It is also recommended to read the [FAQ](https://github.com/MISP/MISP/wiki/Frequently-Asked-Questions)
 
 Contributing
 ------------

From 7bd64bc23f5e780709d8a60dd37d2cefb1e7d671 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 2 Oct 2023 15:30:17 +0200
Subject: [PATCH 34/87] fix: [search] pagination fix

---
 app/Controller/LogsController.php | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/app/Controller/LogsController.php b/app/Controller/LogsController.php
index 1d540ae4893..c58461ff673 100644
--- a/app/Controller/LogsController.php
+++ b/app/Controller/LogsController.php
@@ -273,7 +273,7 @@ public function admin_search($new = false)
             $this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);
 
             // reset the paginate_conditions
-            $this->Session->write('paginate_conditions_log', array());
+            //$this->Session->write('paginate_conditions_log', array());
             if ($this->request->is('post')) {
                 $filters['email'] = $this->request->data['Log']['email'];
                 if (!$orgRestriction) {
@@ -337,12 +337,11 @@ public function admin_search($new = false)
                     $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
                     $this->Session->write('paginate_conditions_log_title', $filters['title']);
                     $this->Session->write('paginate_conditions_log_change', $filters['change']);
-                    $this->Session->write('paginate_conditions_log_change', $filters['from'] ?? null);
-                    $this->Session->write('paginate_conditions_log_change', $filters['to'] ?? null);
+                    $this->Session->write('paginate_conditions_log_from', $filters['from'] ?? null);
+                    $this->Session->write('paginate_conditions_log_to', $filters['to'] ?? null);
                     if (Configure::read('MISP.log_client_ip')) {
                         $this->Session->write('paginate_conditions_log_ip', $filters['ip']);
                     }
-
                     // set the same view as the index page
                     $this->render('index');
                 }
@@ -355,12 +354,11 @@ public function admin_search($new = false)
                 $filters['model_id'] = $this->Session->read('paginate_conditions_log_model_id');
                 $filters['title'] = $this->Session->read('paginate_conditions_log_title');
                 $filters['change'] = $this->Session->read('paginate_conditions_log_change');
-                $filters['change'] = $this->Session->read('paginate_conditions_log_from') ?? null;
-                $filters['change'] = $this->Session->read('paginate_conditions_log_to') ?? null;
+                $filters['from'] = $this->Session->read('paginate_conditions_log_from') ?? null;
+                $filters['to'] = $this->Session->read('paginate_conditions_log_to') ?? null;
                 if (Configure::read('MISP.log_client_ip')) {
                     $filters['ip'] = $this->Session->read('paginate_conditions_log_ip');
                 }
-
                 // for info on what was searched for
                 $this->set('emailSearch', $filters['email']);
                 $this->set('orgSearch', $filters['org']);
@@ -378,7 +376,7 @@ public function admin_search($new = false)
 
                 // re-get pagination
                 $this->{$this->defaultModel}->recursive = 0;
-                $this->paginate = array_merge_recursive($this->Session->read('paginate_conditions_log'), $this->paginate);
+                $this->paginate = array_replace_recursive($this->paginate, $this->Session->read('paginate_conditions_log'));
                 if (!isset($this->paginate['order'])) {
                     $this->paginate['order'] = array('Log.id' => 'DESC');
                 }

From 798255625adc2648ef6578b676e19246fa1ffd9e Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Mon, 2 Oct 2023 16:41:53 +0200
Subject: [PATCH 35/87] fix: [Taxonomies] fix enabling of individual taxonomy
 tags for tags with special chars. fixes ##9300

---
 app/Model/Taxonomy.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/Model/Taxonomy.php b/app/Model/Taxonomy.php
index 4b12616c552..9228546d7a0 100644
--- a/app/Model/Taxonomy.php
+++ b/app/Model/Taxonomy.php
@@ -445,11 +445,11 @@ public function addTags($id, $tagList = false)
             }
             if ($tagList) {
                 foreach ($tagList as $tagName) {
-                    if ($tagName === $entry['tag']) {
+                    if ($tagName === $entry['tag'] || $tagName === h($entry['tag'])) {
                         if (isset($tags[strtoupper($entry['tag'])])) {
-                            $this->Tag->quickEdit($tags[strtoupper($entry['tag'])], $tagName, $colour, 0, $numerical_value);
+                            $this->Tag->quickEdit($tags[strtoupper($entry['tag'])], $entry['tag'], $colour, 0, $numerical_value);
                         } else {
-                            $this->Tag->quickAdd($tagName, $colour, $numerical_value);
+                            $this->Tag->quickAdd($entry['tag'], $colour, $numerical_value);
                         }
                     }
                 }

From 24ac6ae4b87db530b897ae78a91aac87b0625301 Mon Sep 17 00:00:00 2001
From: Tom King <13371308+tking2@users.noreply.github.com>
Date: Tue, 3 Oct 2023 14:41:55 +0100
Subject: [PATCH 36/87] fix: [Attribute REST] Add sharinggroup as an allowed
 parameter for attribute filtering

---
 app/Controller/Component/RestSearchComponent.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index bbc2bd58423..04cd3282ccf 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -52,6 +52,7 @@ class RestSearchComponent extends Component
             'first_seen',
             'last_seen',
             'eventinfo',
+            'sharinggroup',
             'allow_proposal_blocking',
             'flatten',
             'list',

From 02e173b769a0c4e1fcca3397fce5c3377991437e Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Wed, 4 Oct 2023 12:40:04 +0200
Subject: [PATCH 37/87] new: [User] Add setting to limit site admin roles to
 instance's host org.

---
 app/Model/Server.php |  8 ++++++++
 app/Model/User.php   | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 0a369f81f26..c260fbff777 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -6407,6 +6407,14 @@ private function generateServerSettings()
                     'type' => 'boolean',
                     'null' => true
                 ),
+                'limit_site_admins_to_host_org' => array(
+                    'level' => self::SETTING_RECOMMENDED,
+                    'description' => __('If enabled, it will only be possible to assign site admin roles to users belonging to the instance\'s host org.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true
+                ),
                 'disable_browser_cache' => array(
                     'level' => 0,
                     'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'),
diff --git a/app/Model/User.php b/app/Model/User.php
index afbec075ec2..3b8318b18ff 100644
--- a/app/Model/User.php
+++ b/app/Model/User.php
@@ -262,6 +262,16 @@ public function beforeValidate($options = array())
         if (empty($user['nids_sid'])) {
             $user['nids_sid'] = mt_rand(1000000, 9999999);
         }
+        if (!empty(Configure::read('Security.limit_site_admins_to_host_org'))){
+            if (!empty($user['role_id']) and !empty($user['org_id'] and $user['org_id'] != Configure::read('MISP.host_org_id'))){
+                $role = $this->Role->find('first', array(
+                    'conditions' => array('Role.id' => $user['role_id'])
+                ));
+                if (!empty($role) and $role['Role']['perm_site_admin'] === true){
+                    $this->invalidate('role_id', "Site admin roles can only be assigned to users of the host org on this instance.");
+                }
+            }
+        }
         return true;
     }
 

From 00c7a7169ea0b736ae77717a5454730621dce424 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 4 Oct 2023 13:16:33 +0200
Subject: [PATCH 38/87] fix: [security] XSS in selectGalaxy

- As reported by Zigrin Security
---
 app/Controller/GalaxiesController.php | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/Controller/GalaxiesController.php b/app/Controller/GalaxiesController.php
index 89eb639b66d..fdc633d2102 100644
--- a/app/Controller/GalaxiesController.php
+++ b/app/Controller/GalaxiesController.php
@@ -350,14 +350,14 @@ public function selectGalaxy($target_id, $target_type='event', $namespace='misp'
         $items = array(
             array(
                 'name' => __('All clusters'),
-                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . h($local) . '/eventid:' . h($eventid)
             )
         );
         foreach ($galaxies as $galaxy) {
             if (!isset($galaxy['Galaxy']['kill_chain_order']) || $noGalaxyMatrix) {
                 $items[] = array(
                     'name' => h($galaxy['Galaxy']['name']),
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'template' => array(
                         'preIcon' => 'fa-' . $galaxy['Galaxy']['icon'],
                         'name' => $galaxy['Galaxy']['name'],
@@ -367,14 +367,14 @@ public function selectGalaxy($target_id, $target_type='event', $namespace='misp'
             } else { // should use matrix instead
                 $param = array(
                     'name' => $galaxy['Galaxy']['name'],
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'functionName' => sprintf(
                         "getMatrixPopup('%s', '%s', '%s/local:%s/eventid:%s')",
-                        $target_type,
-                        $target_id,
-                        $galaxy['Galaxy']['id'],
-                        $local,
-                        $eventid
+                        h($target_type),
+                        h($target_id),
+                        h($galaxy['Galaxy']['id']),
+                        h($local),
+                        h($eventid)
                     ),
                     'isPill' => true,
                     'isMatrix' => true
@@ -405,12 +405,12 @@ public function selectGalaxyNamespace($target_id, $target_type='event', $noGalax
         $noGalaxyMatrix = $noGalaxyMatrix ? '1' : '0';
         $items = [[
             'name' => __('All namespaces'),
-            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/0' . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/0' . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
         ]];
         foreach ($namespaces as $namespace) {
             $items[] = array(
                 'name' => $namespace,
-                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/' . $namespace . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/' . h($namespace) . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
             );
         }
 

From f8ed3a8d6d3954ee3f700a8310906848345d160c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 4 Oct 2023 13:16:33 +0200
Subject: [PATCH 39/87] fix: [security] XSS in selectGalaxy

- As reported by Zigrin Security
---
 app/Controller/GalaxiesController.php | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/app/Controller/GalaxiesController.php b/app/Controller/GalaxiesController.php
index 89eb639b66d..fdc633d2102 100644
--- a/app/Controller/GalaxiesController.php
+++ b/app/Controller/GalaxiesController.php
@@ -350,14 +350,14 @@ public function selectGalaxy($target_id, $target_type='event', $namespace='misp'
         $items = array(
             array(
                 'name' => __('All clusters'),
-                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . h($local) . '/eventid:' . h($eventid)
             )
         );
         foreach ($galaxies as $galaxy) {
             if (!isset($galaxy['Galaxy']['kill_chain_order']) || $noGalaxyMatrix) {
                 $items[] = array(
                     'name' => h($galaxy['Galaxy']['name']),
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'template' => array(
                         'preIcon' => 'fa-' . $galaxy['Galaxy']['icon'],
                         'name' => $galaxy['Galaxy']['name'],
@@ -367,14 +367,14 @@ public function selectGalaxy($target_id, $target_type='event', $namespace='misp'
             } else { // should use matrix instead
                 $param = array(
                     'name' => $galaxy['Galaxy']['name'],
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'functionName' => sprintf(
                         "getMatrixPopup('%s', '%s', '%s/local:%s/eventid:%s')",
-                        $target_type,
-                        $target_id,
-                        $galaxy['Galaxy']['id'],
-                        $local,
-                        $eventid
+                        h($target_type),
+                        h($target_id),
+                        h($galaxy['Galaxy']['id']),
+                        h($local),
+                        h($eventid)
                     ),
                     'isPill' => true,
                     'isMatrix' => true
@@ -405,12 +405,12 @@ public function selectGalaxyNamespace($target_id, $target_type='event', $noGalax
         $noGalaxyMatrix = $noGalaxyMatrix ? '1' : '0';
         $items = [[
             'name' => __('All namespaces'),
-            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/0' . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/0' . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
         ]];
         foreach ($namespaces as $namespace) {
             $items[] = array(
                 'name' => $namespace,
-                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/' . $namespace . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/' . h($namespace) . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
             );
         }
 

From bcac8674fa719c0c7fd64422c515ef449adb31fb Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Wed, 4 Oct 2023 17:32:58 +0200
Subject: [PATCH 40/87] chg: [RestClient] use http_method value from template
 if available

---
 app/webroot/js/restClient.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/webroot/js/restClient.js b/app/webroot/js/restClient.js
index f2aecf67554..580c8e36a55 100644
--- a/app/webroot/js/restClient.js
+++ b/app/webroot/js/restClient.js
@@ -165,7 +165,11 @@ var debounceTimerUpdate;
             var previously_selected_template = $('#ServerUrl').data('urlWithoutParam')
             if (selected_template !== '' && allValidApis[selected_template] !== undefined) {
                 $('#template_description').show();
-                $('#ServerMethod').val('POST');
+                if(allValidApis[selected_template].http_method !== undefined){
+                    $('#ServerMethod').val(allValidApis[selected_template].http_method);
+                } else {
+                    $('#ServerMethod').val('POST');
+                }
                 var server_url_changed = $('#ServerUrl').val() != allValidApis[selected_template].url;
                 $('#ServerUrl')
                     .val(allValidApis[selected_template].url)

From f7c4d9345d480a447310209e43831dacd4305660 Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Wed, 4 Oct 2023 17:50:43 +0200
Subject: [PATCH 41/87] new: [RestClient] Add user totp_delete to query builder

---
 app/Controller/Component/RestResponseComponent.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
index 0e3f4d09d45..3ae1742ccc7 100644
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@@ -323,6 +323,11 @@ class RestResponseComponent extends Component
                 'description' => 'Simply GET the url endpoint to view the API output of the statistics API. Additional statistics are available via the following tab-options similar to the UI: data, orgs, users, tags, attributehistogram, sightings, attackMatrix',
                 'params' => array('tab'),
                 'http_method' => 'GET'
+            ),
+            'totp_delete' => array(
+                'description' => 'Simply do a DELETE or POST request to the url',
+                'params' => array('user_id'),
+                'http_method' => 'DELETE'
             )
         ),
         'UserSetting' => array(

From 70ee84fdbe2fb29fa2e844a80940ce805c647c4e Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Wed, 4 Oct 2023 19:47:24 +0200
Subject: [PATCH 42/87] chg: [user] Update store api access time setting
 description (#9313)

Api access time is stored once per hour by default (since commit a5f5a4e113872a77d4e6c2b1a125f03ee89773c2), making the old description of this setting incorrect.
---
 app/Model/Server.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index c260fbff777..8d5d5e78f89 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -5615,7 +5615,7 @@ private function generateServerSettings()
                 ),
                 'store_api_access_time' => array(
                     'level' => 1,
-                    'description' => __('If enabled, MISP will capture the last API access time following a successful authentication using API keys, stored against a user under the last_api_access field.'),
+                    'description' => __('If enabled, MISP will capture a users\' last API access time following every successful authentication using API keys (as opposed to once max per hour by default). Stored as last_api_access time for the user.'),
                     'value' => false,
                     'test' => 'testBool',
                     'type' => 'boolean',

From 97a96571ef01ce1047d84cb8a8e339eb23eb9c46 Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Wed, 4 Oct 2023 22:12:31 +0200
Subject: [PATCH 43/87] new: [OpenApi] add doc for user totp_delete endpoint

---
 app/webroot/doc/openapi.yaml | 66 ++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/app/webroot/doc/openapi.yaml b/app/webroot/doc/openapi.yaml
index 7e2ff9e9d2b..aa7187cda20 100644
--- a/app/webroot/doc/openapi.yaml
+++ b/app/webroot/doc/openapi.yaml
@@ -914,6 +914,24 @@ paths:
         default:
           $ref: "#/components/responses/ApiErrorResponse"
 
+  /users/totp_delete/{userId}:
+    delete:
+      summary: "Delete user TOTP"
+      operationId: deleteUserTotp
+      tags:
+        - Users
+      parameters:
+        - $ref: "#/components/parameters/userIdParameter"
+      responses:
+        "200":
+          $ref: "#/components/responses/DeleteUserTotpResponse"
+        "403":
+          $ref: "#/components/responses/UnauthorizedApiErrorResponse"
+        "404":
+          $ref: "#/components/responses/NotFoundUserTotpDeleteResponse"
+        default:
+          $ref: "#/components/responses/ApiErrorResponse"
+
   /admin/organisations/add:
     post:
       summary: "Add organisation"
@@ -5879,6 +5897,23 @@ components:
           type: string
           example: "/attributes/1234"
 
+    NotFoundUserTotpDeleteError:
+      type: object
+      required:
+        - name
+        - message
+        - url
+      properties:
+        name:
+          type: string
+          example: "Invalid user"
+        message:
+          type: string
+          example: "Invalid user"
+        url:
+          type: string
+          example: "/users/totp_delete/1"
+
   parameters:
     eventIdParameter:
       name: eventId
@@ -7573,6 +7608,30 @@ components:
                 type: string
                 example: "/admin/users/delete/1"
 
+    DeleteUserTotpResponse:
+      description: "Delete user TOTP response"
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              saved:
+                type: boolean
+              success:
+                type: boolean
+              name:
+                type: string
+                example: "User TOTP deleted."
+              message:
+                type: string
+                example: "User TOTP deleted."
+              url:
+                type: string
+                example: "/users/totp_delete/1"
+              id:
+                type: string
+                example: "1"
+
     OrganisationResponse:
       description: "Organisation list response"
       content:
@@ -9081,6 +9140,13 @@ components:
           schema:
             $ref: "#/components/schemas/NotFoundApiError"
 
+    NotFoundUserTotpDeleteResponse:
+      description: "The specified resource was not found"
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/NotFoundUserTotpDeleteError"
+
     UnauthorizedApiErrorResponse:
       description: "Authentication failed. Please make sure you pass the API key of an API enabled user along in the Authorization header."
       content:

From a3c728674f50fa8a298ab20f1b5b25bc1d5e7a81 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 5 Oct 2023 17:05:22 +0200
Subject: [PATCH 44/87] chg: [event restsearch] exposed
 includeGranularCorrelations

- and also made it visible in the JSON output
---
 .../Component/RestSearchComponent.php         |  3 ++-
 app/Lib/Tools/JSONConverterTool.php           | 20 +++++++++++++++++++
 app/Model/Event.php                           |  1 +
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index bbc2bd58423..420fdd82c51 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -119,7 +119,8 @@ class RestSearchComponent extends Component
             'noEventReports',
             'noShadowAttributes',
             'order',
-            'protected'
+            'protected',
+            'includeGranularCorrelations'
         ],
         'Object' => [
             'returnFormat',
diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php
index fe4b2427938..48a93eb0e26 100644
--- a/app/Lib/Tools/JSONConverterTool.php
+++ b/app/Lib/Tools/JSONConverterTool.php
@@ -79,11 +79,31 @@ public static function convert($event, $isSiteAdmin=false, $raw = false)
         }
         if (isset($event['Event']['Attribute'])) {
             $event['Event']['Attribute'] = self::__cleanAttributes($event['Event']['Attribute'], $tempSightings);
+            if (!empty($event['Event']['RelatedAttribute'])) {
+                foreach ($event['Event']['Attribute'] as $k => $attribute) {
+                    if (isset($event['Event']['RelatedAttribute'][$attribute['id']])) {
+                        foreach($event['Event']['RelatedAttribute'][$attribute['id']] as $correlation) {
+                            $event['Event']['Attribute'][$k]['RelatedAttribute'][] = [
+                                'id' => $correlation['attribute_id'],
+                                'value' => $correlation['value'],
+                                'org_id' => $correlation['org_id'],
+                                'info' => $correlation['info'],
+                                'event_id' => $correlation['id']
+                            ];
+                        }
+                    }
+                }
+            }
         }
         if (isset($event['Event']['Object'])) {
             $event['Event']['Object'] = self::__cleanObjects($event['Event']['Object'], $tempSightings);
         }
         unset($tempSightings);
+        if (!empty($event['Event']['RelatedAttribute'])) {
+            foreach ($event['Event']['RelatedAttribute'] as $attribute_id => $relatedAttribute) {
+
+            }
+        }
         unset($event['Event']['RelatedAttribute']);
 
         // Remove information about user_id from JSON export
diff --git a/app/Model/Event.php b/app/Model/Event.php
index e248465bb89..fe43472650e 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -178,6 +178,7 @@ class Event extends AppModel
         'includeFeedCorrelations',
         'includeServerCorrelations',
         'includeWarninglistHits',
+        'includeGranularCorrelations',
         'noEventReports', // do not include event report in event data
         'noShadowAttributes', // do not fetch proposals,
         'limit',

From 03ccdbac70ff65c9315a73027e5aac42ad602cbd Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Fri, 6 Oct 2023 15:32:48 +0200
Subject: [PATCH 45/87] fix: [restsearch] granular correlation parameter fixes

---
 app/Lib/Tools/JSONConverterTool.php | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php
index 48a93eb0e26..7ede8dd2530 100644
--- a/app/Lib/Tools/JSONConverterTool.php
+++ b/app/Lib/Tools/JSONConverterTool.php
@@ -97,13 +97,25 @@ public static function convert($event, $isSiteAdmin=false, $raw = false)
         }
         if (isset($event['Event']['Object'])) {
             $event['Event']['Object'] = self::__cleanObjects($event['Event']['Object'], $tempSightings);
-        }
-        unset($tempSightings);
-        if (!empty($event['Event']['RelatedAttribute'])) {
-            foreach ($event['Event']['RelatedAttribute'] as $attribute_id => $relatedAttribute) {
-
+            if (!empty($event['Event']['RelatedAttribute'])) {
+                foreach ($event['Event']['Object'] as $k => $object) {
+                    foreach ($event['Event']['Attribute'] as $k2 => $attribute) {
+                        if (isset($event['Event']['RelatedAttribute'][$attribute['id']])) {
+                            foreach($event['Event']['RelatedAttribute'][$attribute['id']] as $correlation) {
+                                $event['Event']['Object'][$k]['Attribute'][$k2]['RelatedAttribute'][] = [
+                                    'id' => $correlation['attribute_id'],
+                                    'value' => $correlation['value'],
+                                    'org_id' => $correlation['org_id'],
+                                    'info' => $correlation['info'],
+                                    'event_id' => $correlation['id']
+                                ];
+                            }
+                        }
+                    }
+                }
             }
         }
+        unset($tempSightings);
         unset($event['Event']['RelatedAttribute']);
 
         // Remove information about user_id from JSON export

From 20ae70d2d0b018bc2c3c5ee178b066664f3a49e6 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 6 Oct 2023 15:40:24 +0200
Subject: [PATCH 46/87] fix: [eventreport:extractEntities] Enforced minimum
 amount of char for valid replacement

---
 app/Model/EventReport.php | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php
index 1ab7f3dfe4b..6e8775797ef 100644
--- a/app/Model/EventReport.php
+++ b/app/Model/EventReport.php
@@ -643,6 +643,9 @@ public function transformFreeTextIntoReplacement(array $user, array $report, arr
         foreach ($proxyElements['attribute'] as $uuid => $attribute) {
             $count = 0;
             $textToInject = sprintf('@[attribute](%s)', $uuid);
+            if (strlen($attribute['value']) < 3) {
+                continue;
+            }
             $content = str_replace($attribute['value'], $textToInject, $content, $count);
             if ($count > 0 || strpos($originalContent, $attribute['value'])) { // Check if the value has been replaced by the first match
                 if (!isset($replacedValues[$attribute['value']])) {
@@ -784,6 +787,9 @@ public function extractWithReplacements(array $user, array $report, array $optio
             $tags = $this->Tag->fetchUsableTags($user);
             foreach ($tags as $tag) {
                 $tagName = $tag['Tag']['name'];
+                if (strlen($tagName) < 3) {
+                    continue;
+                }
                 $found = $this->isValidReplacementTag($content, $tagName);
                 if ($found) {
                     $replacedContext[$tagName][$tagName] = $tag['Tag'];

From d72fbb7c45a1da056be25b5a521d8d2e902724a7 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 6 Oct 2023 15:40:24 +0200
Subject: [PATCH 47/87] fix: [eventreport:extractEntities] Enforced minimum
 amount of char for valid replacement

---
 app/Model/EventReport.php | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php
index 1ab7f3dfe4b..6e8775797ef 100644
--- a/app/Model/EventReport.php
+++ b/app/Model/EventReport.php
@@ -643,6 +643,9 @@ public function transformFreeTextIntoReplacement(array $user, array $report, arr
         foreach ($proxyElements['attribute'] as $uuid => $attribute) {
             $count = 0;
             $textToInject = sprintf('@[attribute](%s)', $uuid);
+            if (strlen($attribute['value']) < 3) {
+                continue;
+            }
             $content = str_replace($attribute['value'], $textToInject, $content, $count);
             if ($count > 0 || strpos($originalContent, $attribute['value'])) { // Check if the value has been replaced by the first match
                 if (!isset($replacedValues[$attribute['value']])) {
@@ -784,6 +787,9 @@ public function extractWithReplacements(array $user, array $report, array $optio
             $tags = $this->Tag->fetchUsableTags($user);
             foreach ($tags as $tag) {
                 $tagName = $tag['Tag']['name'];
+                if (strlen($tagName) < 3) {
+                    continue;
+                }
                 $found = $this->isValidReplacementTag($content, $tagName);
                 if ($found) {
                     $replacedContext[$tagName][$tagName] = $tag['Tag'];

From 3d57ce7aa0b67667aa059775a58f689ca22c3535 Mon Sep 17 00:00:00 2001
From: Luciano Righetti <luciano.righetti@gmail.com>
Date: Fri, 6 Oct 2023 15:56:30 +0200
Subject: [PATCH 48/87] fix: wrong param order fixes #9319

---
 app/View/Elements/Events/View/row_attribute.ctp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/View/Elements/Events/View/row_attribute.ctp b/app/View/Elements/Events/View/row_attribute.ctp
index 595f3d7d787..33d09fc9ada 100644
--- a/app/View/Elements/Events/View/row_attribute.ctp
+++ b/app/View/Elements/Events/View/row_attribute.ctp
@@ -346,7 +346,7 @@
               endif;
               if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
         ?>
-          <span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Enrichment/0/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
+          <span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/0/Enrichment/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
         <?php
               endif;
         ?>

From 54ec65acb2d2332702772057f97dbc4e97882b7c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 9 Oct 2023 14:24:31 +0200
Subject: [PATCH 49/87] chg: [workflows:infoModal] Added link to jinja2
 official documentation

---
 app/View/Elements/Workflows/infoModal.ctp | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/app/View/Elements/Workflows/infoModal.ctp b/app/View/Elements/Workflows/infoModal.ctp
index e5179362d3c..24729bede3b 100644
--- a/app/View/Elements/Workflows/infoModal.ctp
+++ b/app/View/Elements/Workflows/infoModal.ctp
@@ -645,20 +645,21 @@ $data_passed_to_if_module = [
             <div class="tab-pane" id="modal-jinja2">
                 <h3><?= __('Jinja2 Syntax') ?></h3>
                 <p><i class="fa-fw <?= $this->FontAwesome->getClass('exclamation-triangle') ?>"></i> <?= __('For these examples, we consider the module received data under the MISP core format.') ?></p>
+                <p><i class="fa-fw <?= $this->FontAwesome->getClass('link') ?>"></i> <?= __('More documenation available on Jinja2 template designer documentation\'s') ?> <a href="https://jinja.palletsprojects.com/en/3.1.x/templates/"><?= __('website') ?></a></p>
                 <h4><?= __('You can use the dot <code>`.` </code> notation or the subscript syntax <code>`[]`</code> to access attributes of a variable') ?></h4>
                 <ul>
                     <li><code>{{ Event.info }}</code>: <?= __('Shows the title of the event') ?></li>
                     <li><code>{{ Event['info'] }}</code>: <?= __('Shows the title of the event') ?></li>
                 </ul>
                 <h4><?= __('Jinja2 allows you to easily create list') ?></h4>
-<pre>
+                <pre>
 {% for attribute in Event.Attribute %}
 - {{ attribute.value }}
 {% endfor %}
 </pre>
 
                 <h4><?= __('Jinja2 allows you to add logic') ?></h4>
-<pre>
+                <pre>
 {% if "tlp:white" in Event.Tag %}
     - This Event has the TLP:WHITE tag
 {% else %}
@@ -667,7 +668,7 @@ $data_passed_to_if_module = [
 </pre>
 
                 <h4><?= __('Jinja2 allows you to modify variables by using filters') ?></h4>
-<pre>
+                <pre>
 # The `reverse` filter
 - `{{ Event.info | reverse }}`
 -> The event title, but reversed
@@ -682,11 +683,15 @@ $data_passed_to_if_module = [
     - {{ attribute.value }}
     {% endfor %}
 {% endfor %}
+
+# The `json` filter
+{{ attribute | tojson }}
+-> The complete attribute json encoded
 </pre>
 
             </div>
 
-            <div class="tab-pane" id="modal-blueprint">
+            <div class=" tab-pane" id="modal-blueprint">
                 <h3><?= __('Blueprints') ?></h3>
                 <ul>
                     <li><?= __('Blueprints allow user to saved a collection of modules and how they are connected together so that they can be re-used and shared.') ?></li>

From a1005dc55165e30c59220e4f506516c51a2ce72c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 9 Oct 2023 14:25:10 +0200
Subject: [PATCH 50/87] chg: [tools:misp-workflows/webhook_listener] Print
 headers on stdout

---
 tools/misp-workflows/webhook-listener.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/misp-workflows/webhook-listener.py b/tools/misp-workflows/webhook-listener.py
index 400bb94ae50..3bde8f1cbef 100755
--- a/tools/misp-workflows/webhook-listener.py
+++ b/tools/misp-workflows/webhook-listener.py
@@ -23,6 +23,7 @@ def do_POST(self):
         self.send_response(200)
         self.send_header("Content-type", "text/html")
         self.end_headers()
+        pprint(dict(self.headers))
         self.data_string = self.rfile.read(int(self.headers['Content-Length']))
         self.data_string = self.data_string.decode('utf8')
         try:

From fc8cf9ee8c4d36977aa68d99f2c4688e83bac332 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 9 Oct 2023 14:25:46 +0200
Subject: [PATCH 51/87] fix: [workflow:baseModule] Only build fast lookup array
 if trigger is using the MISP core format

---
 app/Model/WorkflowModules/WorkflowBaseModule.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php
index 23da2a3ee6b..620fd4d3684 100644
--- a/app/Model/WorkflowModules/WorkflowBaseModule.php
+++ b/app/Model/WorkflowModules/WorkflowBaseModule.php
@@ -314,7 +314,9 @@ class WorkflowBaseActionModule extends WorkflowBaseModule
     public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
     {
         $rData = $roamingData->getData();
-        $this->_buildFastLookupForRoamingData($rData);
+        if ($this->expect_misp_core_format) {
+            $this->_buildFastLookupForRoamingData($rData);
+        }
         return true;
     }
 

From 05714df9d4d51a67ae133db8cd2103312e53ef3b Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 9 Oct 2023 14:27:37 +0200
Subject: [PATCH 52/87] chg: [workflow-module:webhook] Added support of
 request_method, headers and payload

Partially fix #9321
---
 .../action/Module_ms_teams_webhook.php        |  4 +-
 .../WorkflowModules/action/Module_webhook.php | 66 +++++++++++++++----
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
index 4960f094a1f..7708c766929 100644
--- a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
@@ -5,7 +5,7 @@ class Module_ms_teams_webhook extends Module_webhook
 {
     public $id = 'ms-teams-webhook';
     public $name = 'MS Teams Webhook';
-    public $version = '0.4';
+    public $version = '0.5';
     public $description = 'Perform callbacks to the MS Teams webhook provided by the "Incoming Webhook" connector';
     public $icon_path = 'MS_Teams.png';
 
@@ -38,7 +38,7 @@ public function __construct()
         ];
     }
 
-    protected function doRequest($url, $contentType, $data, $headers = [], $serverConfig = null)
+    protected function doRequest($url, $contentType, $data, $headers = [], $requestMethod='post', $serverConfig = null)
     {
         $data = ['text' => JsonTool::encode($data)];
         return parent::doRequest($url, $contentType, $data);
diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php
index 9baf9500ea9..c9787ce0910 100644
--- a/app/Model/WorkflowModules/action/Module_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_webhook.php
@@ -8,7 +8,7 @@ class Module_webhook extends WorkflowBaseActionModule
 {
     public $id = 'webhook';
     public $name = 'Webhook';
-    public $version = '0.4';
+    public $version = '0.5';
     public $description = 'Allow to perform custom callbacks to the provided URL';
     public $icon_path = 'webhook.png';
     public $inputs = 1;
@@ -25,7 +25,7 @@ public function __construct()
         $this->params = [
             [
                 'id' => 'url',
-                'label' => __('Payload URL'),
+                'label' => __('URL'),
                 'type' => 'input',
                 'placeholder' => 'https://example.com/test',
             ],
@@ -40,11 +40,29 @@ public function __construct()
                 ],
             ],
             [
-                'id' => 'data_extraction_path',
-                'label' => __('Data extraction path'),
-                'type' => 'hashpath',
+                'id' => 'request_method',
+                'label' => __('HTTP Request Method'),
+                'type' => 'select',
+                'default' => 'post',
+                'options' => [
+                    'post' => 'POST',
+                    'get' => 'GET',
+                    'put' => 'PUT',
+                    'delete' => 'DELETE',
+                ],
+            ],
+            [
+                'id' => 'payload',
+                'label' => __('Payload (leave empty for roaming data)'),
+                'type' => 'textarea',
                 'default' => '',
-                'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
+                'placeholder' => '',
+            ],
+            [
+                'id' => 'headers',
+                'label' => __('Headers'),
+                'type' => 'textarea',
+                'placeholder' => 'Authorization: foobar',
             ],
         ];
     }
@@ -82,10 +100,20 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
         }
 
         $rData = $roamingData->getData();
-        $path = $params['data_extraction_path']['value'];
-        $extracted = !empty($params['data_extraction_path']['value']) ? $this->extractData($rData, $path) : $rData;
+        $payload = '';
+        if (strlen($params['payload']['value']) > 0) {
+            $payload = $this->render_jinja_template($params['payload']['value'], $rData);
+        } else {
+            $payload = $rData;
+        }
+        $tmpHeaders = explode(PHP_EOL, $params['headers']['value']);
+        $headers = [];
+        foreach ($tmpHeaders as $entry) {
+            $entry = explode(':', $entry, 2);
+            $headers[trim($entry[0])] = trim($entry[1]);
+        }
         try {
-            $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $extracted);
+            $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $payload, $headers, $params['request_method']['value']);
             if ($response->isOk()) {
                 return true;
             }
@@ -106,7 +134,7 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
         return false;
     }
 
-    protected function doRequest($url, $contentType, $data, $headers = [], $serverConfig = null)
+    protected function doRequest($url, $contentType, $data, $headers = [], $requestMethod='post', $serverConfig = null)
     {
         $this->Event = ClassRegistry::init('Event'); // We just need a model to use AppModel functions
         $version = implode('.', $this->Event->checkMISPVersion());
@@ -122,11 +150,25 @@ protected function doRequest($url, $contentType, $data, $headers = [], $serverCo
         $syncTool = new SyncTool();
         $serverConfig = !empty($serverConfig['Server']) ? $serverConfig : ['Server' => $serverConfig];
         $HttpSocket = $syncTool->setupHttpSocket($serverConfig, $this->timeout);
+        $encodedData = $data;
         if ($contentType == 'form') {
             $request['header']['Content-Type'] = 'application/x-www-form-urlencoded';
-            $response = $HttpSocket->post($url, $data, $request);
         } else {
-            $response = $HttpSocket->post($url, JsonTool::encode($data), $request);
+            $encodedData = JsonTool::encode($data);
+        }
+        switch ($requestMethod) {
+            case 'post':
+                $response = $HttpSocket->post($url, $encodedData, $request);
+                break;
+            case 'get':
+                $response = $HttpSocket->get($url, false, $request);
+                break;
+            case 'put':
+                $response = $HttpSocket->put($url, $encodedData, $request);
+                break;
+            case 'delete':
+                $response = $HttpSocket->delete($url, $encodedData, $request);
+                break;
         }
         return $response;
     }

From 79905a0eb81622603a4c26a2374c101d14894465 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 10 Oct 2023 08:56:06 +0200
Subject: [PATCH 53/87] chg: [workflow] Improved logging and debugging of
 workflow execution

---
 app/Model/Workflow.php | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index acb10df1229..4d2eb7b966b 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -532,6 +532,9 @@ private function __runWorkflow(array $workflow, $triggerModule, array $data, $st
         if (empty($blockingPathExecutionSuccess)) {
             $message = __('Execution stopped. %s', PHP_EOL . implode(', ', $blockingErrors));
             $this->logExecutionError($workflow, $message);
+        } else if (!empty($blockingErrors)) {
+            $message = __('Execution encountered an error but continued. %s', PHP_EOL . implode(', ', $blockingErrors));
+            $this->logExecutionError($workflow, $message);
         }
         $outcomeText = 'failure';
         if (!empty($blockingPathExecutionSuccess)) {
@@ -663,8 +666,12 @@ public function executeNode(array $node, WorkflowRoamingData $roamingData, array
             return false;
         }
         if (!is_null($moduleClass)) {
+            $execErrors = [];
             try {
-                $success = $moduleClass->exec($node, $roamingData, $errors);
+                $success = $moduleClass->exec($node, $roamingData, $execErrors);
+                if (!empty($execErrors)) {
+                    $errors += $execErrors;
+                }
             } catch (Exception $e) {
                 $message = __('Error while executing module %s. Error: %s', $node['data']['id'], $e->getMessage());
                 $this->logExecutionError($roamingData->getWorkflow(), $message);
@@ -679,15 +686,23 @@ public function executeNode(array $node, WorkflowRoamingData $roamingData, array
             $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
             return false;
         }
-        $message = __('Executed node `%s`' .  PHP_EOL . 'Node `%s` (%s) from Workflow `%s` (%s) executed successfully',
+        $sucessType = $success ? 'success' : 'partial-success';
+        $message = __('Executed node `%s`' .  PHP_EOL . 'Node `%s` (%s) from Workflow `%s` (%s) executed successfully with status: %s',
             $node['data']['id'],
             $node['data']['id'],
             $node['id'],
             $roamingData->getWorkflow()['Workflow']['name'],
-            $roamingData->getWorkflow()['Workflow']['id']
+            $roamingData->getWorkflow()['Workflow']['id'],
+            $sucessType
         );
         $this->logExecutionIfDebug($roamingData->getWorkflow(), $message);
-        $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());
+        $this->sendRequestToDebugEndpointIfDebug(
+            $roamingData->getWorkflow(),
+            $node,
+            sprintf('/exec/%s?result=%s', $moduleClass->id, $sucessType),
+            $roamingData->getData(),
+            $execErrors
+        );
         return $success;
     }
 
@@ -1495,14 +1510,14 @@ private function __saveAndReturnErrors($data, $saveOptions = [], &$errors = [])
         return $saveSuccess;
     }
 
-    public function sendRequestToDebugEndpointIfDebug(array $workflow, array $node, $path='/', array $data=[])
+    public function sendRequestToDebugEndpointIfDebug(array $workflow, array $node, $path='/', array $data=[], array $errors=[])
     {
         if ($workflow['Workflow']['debug_enabled']) {
-            $this->sendRequestToDebugEndpoint($workflow, $node, $path, $data);
+            $this->sendRequestToDebugEndpoint($workflow, $node, $path, $data, $errors);
         }
     }
 
-    public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='/', array $data=[])
+    public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='/', array $data=[],  array $errors=[])
     {
         $debug_url = Configure::read('Plugin.Workflow_debug_url');
         App::uses('HttpSocket', 'Network/Http');
@@ -1520,6 +1535,9 @@ public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='
             'timestamp' => date("c"),
             'data' => $data,
         ];
+        if (!empty($errors)) {
+            $dataToPost['errors'] = $errors;
+        }
         $socket->post($uri, JsonTool::encode($dataToPost));
     }
     

From c1be0c8ae74f9187728d494722743c4f3d7f15b6 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 10 Oct 2023 08:58:14 +0200
Subject: [PATCH 54/87] fix: [workflowModules:webhook] Gracefully handle case
 with empty headers

---
 app/Model/WorkflowModules/action/Module_webhook.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php
index c9787ce0910..c98b7c7aa73 100644
--- a/app/Model/WorkflowModules/action/Module_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_webhook.php
@@ -110,7 +110,9 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
         $headers = [];
         foreach ($tmpHeaders as $entry) {
             $entry = explode(':', $entry, 2);
-            $headers[trim($entry[0])] = trim($entry[1]);
+            if (count($entry) == 2) {
+                $headers[trim($entry[0])] = trim($entry[1]);
+            }
         }
         try {
             $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $payload, $headers, $params['request_method']['value']);

From 8e82260524f0e76e906bc6153ab9420cc0d54ff7 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 10 Oct 2023 08:58:56 +0200
Subject: [PATCH 55/87] chg: [workfowModules:webhook] Added support of
 self-signed certificates

---
 .../WorkflowModules/action/Module_webhook.php     | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php
index c98b7c7aa73..717b063266d 100644
--- a/app/Model/WorkflowModules/action/Module_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_webhook.php
@@ -8,7 +8,7 @@ class Module_webhook extends WorkflowBaseActionModule
 {
     public $id = 'webhook';
     public $name = 'Webhook';
-    public $version = '0.5';
+    public $version = '0.6';
     public $description = 'Allow to perform custom callbacks to the provided URL';
     public $icon_path = 'webhook.png';
     public $inputs = 1;
@@ -51,6 +51,16 @@ public function __construct()
                     'delete' => 'DELETE',
                 ],
             ],
+            [
+                'id' => 'self_signed',
+                'label' => __('Self-signed certificates'),
+                'type' => 'select',
+                'default' => 'deny',
+                'options' => [
+                    'deny' => 'Deny self-signed certificates',
+                    'allow' => 'Allow self-signed certificates',
+                ],
+            ],
             [
                 'id' => 'payload',
                 'label' => __('Payload (leave empty for roaming data)'),
@@ -108,6 +118,7 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
         }
         $tmpHeaders = explode(PHP_EOL, $params['headers']['value']);
         $headers = [];
+        $selfSignedAllowed = $params['self_signed']['value'] == 'allow';
         foreach ($tmpHeaders as $entry) {
             $entry = explode(':', $entry, 2);
             if (count($entry) == 2) {
@@ -115,7 +126,7 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
             }
         }
         try {
-            $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $payload, $headers, $params['request_method']['value']);
+            $response = $this->doRequest($params['url']['value'], $params['content_type']['value'], $payload, $headers, $params['request_method']['value'], ['self_signed' => $selfSignedAllowed]);
             if ($response->isOk()) {
                 return true;
             }

From 438d1b8a4333bb2fee31d89a56addd6367638f42 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 11 Oct 2023 09:35:07 +0200
Subject: [PATCH 56/87] chg: [workflow:editor] Improved quick hashpath picker

---
 app/View/Elements/Workflows/infoModal.ctp          |  6 ++++++
 .../js/workflows-editor/workflows-editor.js        | 14 +++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/app/View/Elements/Workflows/infoModal.ctp b/app/View/Elements/Workflows/infoModal.ctp
index 24729bede3b..485d178602a 100644
--- a/app/View/Elements/Workflows/infoModal.ctp
+++ b/app/View/Elements/Workflows/infoModal.ctp
@@ -252,6 +252,9 @@ $data_passed_to_if_module = [
                         "relationship_type": null,
                         "inherited": false
                     }
+                ],
+                "enrichment": [
+                    {}
                 ]
             }
         ],
@@ -580,6 +583,9 @@ $data_passed_to_if_module = [
                         "relationship_type": null,
                         "inherited": false
                     }
+                ],
+                "enrichment": [
+                    {}
                 ]
             },
             {
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 3134917d151..137a2e1036d 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -143,11 +143,18 @@ var iconBySeverity = {
     'error': 'fa-exclamation-circle',
 }
 var severities = ['info', 'warning', 'error']
-var haspathQuickPickMenu = [
+var haspathQuickPickMenuElementSelector = [
     { 'name': 'All Attributes', 'path': 'Event._AttributeFlattened.{n}' },
     { 'name': 'All tags attached to all Attributes', 'path': 'Event._AttributeFlattened.{n}.Tag.{n}.name' },
     { 'name': 'All tags attached to the Event', 'path': 'Event.Tag.{n}.name' },
 ]
+var haspathQuickPickMenuSubElementSelector = [
+    { 'name': 'Attribute type', 'path': 'type' },
+    { 'name': 'All tags', 'path': 'Tag.{n}.name' },
+    { 'name': 'Warnings from warninglists', 'path': 'warnings.{n}.warninglist_category' },
+    { 'name': 'Feed correlation', 'path': 'Feed.{n}.name' },
+    { 'name': 'All enrichments', 'path': 'enrichment.{n}' },
+]
 
 var workflow_id = 0
 var contentChanged = false
@@ -1852,7 +1859,8 @@ function genInput(options, isTextArea, forNode = true) {
 }
 
 function genHashpathInput(options, forNode = true) {
-    function hashPathGenDropdownMenu() {
+    function hashPathGenDropdownMenu(hashpathOptions) {
+        var haspathQuickPickMenu = hashpathOptions.is_sub_selector ? haspathQuickPickMenuSubElementSelector : haspathQuickPickMenuElementSelector
         var $divider = $('<li>').addClass('divider')
         var $dropdownMenu = $('<ul>').addClass('dropdown-menu pull-right')
         var $liPicker = $('<li>').append(
@@ -1921,7 +1929,7 @@ function genHashpathInput(options, forNode = true) {
         .attr('data-toggle', 'dropdown')
         .text('Pick ')
         .append($('<span>').addClass('caret'))
-    var $dropdownMenu = hashPathGenDropdownMenu()
+    var $dropdownMenu = hashPathGenDropdownMenu(options.hashpath ? options.hashpath : {})
     $dropdownContainer.append($dropdownButton, $dropdownMenu)
     $addonContainer.append($input, $dropdownContainer)
     $container.append($label, $addonContainer)

From ed56c40b49bad005711135e5f4c4bee3a86d7f0f Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 11 Oct 2023 09:36:40 +0200
Subject: [PATCH 57/87] chg: [workflow-module:generic_filter] Improved support
 of quick hashpath filter

---
 .../WorkflowModules/logic/Module_generic_filter_data.php  | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index 581a029b952..8324f58ed7e 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -35,8 +35,11 @@ public function __construct()
             [
                 'id' => 'selector',
                 'label' => __('Data selector'),
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'Event._AttributeFlattened.{n}',
+                'hashpath' => [
+                    'is_sub_selector' => false
+                ]
             ],
             [
                 'id' => 'value',
@@ -69,6 +72,9 @@ public function __construct()
                 'label' => __('Hash path'),
                 'type' => 'hashpath',
                 'placeholder' => 'Tag.name',
+                'hashpath' => [
+                    'is_sub_selector' => true
+                ]
             ],
         ];
     }

From 11ccb4ab0c58defbffd7424d083bc06bee5e8c9c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 11 Oct 2023 09:36:57 +0200
Subject: [PATCH 58/87] new: [workflow:editor] Added option to provide a custom
 JSON in the hashpath picker helper

---
 .../js/workflows-editor/workflows-editor.js   | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 137a2e1036d..fccd2af291a 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1649,10 +1649,30 @@ function enableHashpathPicker() {
     })
 }
 
+function redrawFormatPicker(json, associatedParamId) {
+    var jsonData = JSON.parse(json)
+    var $customDataInput = genCustomDataInputForHashpathPicker(associatedParamId)
+    var UIPicker = generateCoreFormatUI(jsonData, associatedParamId)
+    var $modalBody = $('<div>').attr('style', 'display: flex; flex-direction: column').append($customDataInput, UIPicker)
+    $('#core-format-picker').parent().html($modalBody[0])
+}
+
+function genCustomDataInputForHashpathPicker(associatedParamId) {
+    return $('<input>')
+        .attr({
+            id: 'hashpath-custom-format-input',
+            type: 'text',
+            placeholder: 'Provide a custom JSON',
+            onchange: 'redrawFormatPicker(this.value, "' + associatedParamId + '")',
+            style: 'flex-grow: 1; width: unset;'
+        })
+}
+
 function toggleCoreFormatPicker(btn) {
     var associatedParamId = $(btn).closest('.input-append').find('input').data('paramid')
     var sample = JSON.parse($('#misp-core-format-sample').text())
     var UIPicker = generateCoreFormatUI(sample, associatedParamId)
+    var $customDataInput = genCustomDataInputForHashpathPicker(associatedParamId)
     var $selectedPath = $('<input>')
         .attr({
             id: 'selected-hashpath-input',
@@ -1690,7 +1710,8 @@ function toggleCoreFormatPicker(btn) {
         .append($selectedPathOperators, $selectedPath)
     var $closeButton = $('<div>').append($('<a href="#" class="btn" data-dismiss="modal">Close</a>'))
     var $footer = $('<div>').css({display: 'flex'}).append($pathGroup, $closeButton)
-    openModal('Pick Hash path', UIPicker[0].outerHTML, $footer[0].outerHTML, undefined, undefined, 'max-height: 70vh;', 'modal-lg')
+    var $modalBody = $('<div>').attr('style', 'display: flex; flex-direction: column').append($customDataInput, UIPicker)
+    openModal('Pick Hash path', $modalBody[0].outerHTML, $footer[0].outerHTML, undefined, undefined, 'max-height: 70vh;', 'modal-lg')
 }
 
 function genParameterWarning(options) {

From fb61238ee6f0e4c370c5ff4898735c4cda39d63e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 11 Oct 2023 11:15:05 +0200
Subject: [PATCH 59/87] chg: [workflow:executeNode] Correctly interpret
 execution result for logic module when logging

---
 app/Model/Workflow.php | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 4d2eb7b966b..b75e5266121 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -686,7 +686,11 @@ public function executeNode(array $node, WorkflowRoamingData $roamingData, array
             $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
             return false;
         }
-        $sucessType = $success ? 'success' : 'partial-success';
+        if (!empty($this->loaded_modules['logic'][$moduleClass->id])) { // IF module return false for the 2 output.
+            $sucessType = 'success';
+        } else {
+            $sucessType = $success ? 'success' : 'partial-success';
+        }
         $message = __('Executed node `%s`' .  PHP_EOL . 'Node `%s` (%s) from Workflow `%s` (%s) executed successfully with status: %s',
             $node['data']['id'],
             $node['data']['id'],

From 082bda3ae332a84d928500296ccaa7100049be76 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 11 Oct 2023 15:58:46 +0200
Subject: [PATCH 60/87] chg: [layout] Improved UI in special case

---
 app/View/Elements/global_menu.ctp |  18 ++-
 app/View/Elements/hal-ee.ctp      | 193 ++++++++++++++++++++++++++++++
 2 files changed, 210 insertions(+), 1 deletion(-)
 create mode 100644 app/View/Elements/hal-ee.ctp

diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp
index 0771b6529b5..a11e37159b2 100755
--- a/app/View/Elements/global_menu.ctp
+++ b/app/View/Elements/global_menu.ctp
@@ -540,6 +540,18 @@
             )
         );
     }
+    $isHal = date('Y-10-31') == date('Y-m-d');
+    if ($isHal) {
+        $tmp = [
+            'type' => 'root',
+            'url'=> '#',
+            'html' => '<span onclick="toggleHal()" class="fa-stack fa-1x">
+                           <i class="fas fa-broom fa-stack-1x"></i>
+                           <i class="fas fa-cat fa-stack-1x fa-flip-horizontal" style="bottom: 8px; left: 2px;"></i>
+                       </span>'
+        ];
+        $menu_right = array_merge([$tmp], $menu_right);
+    }
 ?>
 <div id="topBar" class="navbar navbar-inverse <?= isset($debugMode) ? $debugMode : 'debugOff' ?>" style="z-index:100;">
   <div class="navbar-inner">
@@ -561,5 +573,9 @@
             }
         ?>
     </ul>
-  </div>
 </div>
+  <?php
+    if ($isHal) {
+        echo $this->element('hal-ee');
+    }
+  ?>
diff --git a/app/View/Elements/hal-ee.ctp b/app/View/Elements/hal-ee.ctp
new file mode 100644
index 00000000000..080482e2b56
--- /dev/null
+++ b/app/View/Elements/hal-ee.ctp
@@ -0,0 +1,193 @@
+<!-- Credit: https://codepen.io/kleberbaum-the-sasster/pen/qBqzVBJ -->
+<div id="hal-div-container" style="display: none;">
+    <div class="loader">
+        <div class="loader-bg">
+            <span></span>
+        </div>
+        <div class="drops drops1">
+            <div class="drop1"></div>
+            <div class="drop2"></div>
+        </div>
+        <div class="drops drops2">
+            <div class="drop1"></div>
+            <div class="drop2"></div>
+        </div>
+        <div class="drops drops3">
+            <div class="drop1"></div>
+            <div class="drop2"></div>
+        </div>
+    </div>
+    <svg xmlns="http://www.w3.org/2000/svg" version="1.1">
+        <defs>
+            <filter id="liquid">
+                <feGaussianBlur in="SourceGraphic" stdDeviation="10" result="blur" />
+                <feColorMatrix in="blur" mode="matrix" values="1 0 0 0 0  0 1 0 0 0  0 0 1 0 0  0 0 0 18 -7" result="liquid" />
+            </filter>
+        </defs>
+    </svg>
+</div>
+
+<script>
+    function toggleHal() {
+        $('#hal-div-container').toggle()
+        $('.navbar-inner').toggleClass('hal-active')
+    }
+</script>
+
+<style>
+    @keyframes fade-in {
+        0% {
+            opacity: 0;
+        }
+
+        100% {
+            opacity: 1;
+        }
+    }
+
+    @keyframes drop {
+        0% {
+            bottom: 0px;
+            opacity: 1;
+        }
+
+        80% {
+            opacity: 1;
+        }
+
+        90% {
+            opacity: 1;
+            bottom: -100px;
+        }
+
+        100% {
+            opacity: 0;
+            bottom: -100px;
+        }
+    }
+
+    @keyframes wave {
+        0% {
+            background-position: 0 160px;
+            background-size: 170px 300px;
+        }
+
+        100% {
+            background-position: 500px -18px;
+            background-size: 250px 150px;
+        }
+    }
+
+    .kuhl {
+        margin-top: 5px;
+        width: 88%;
+    }
+
+    :root {
+        --halloween-color: #ae090f;
+    }
+
+    #topBar.navbar > .navbar-inner.hal-active {
+        background-color: var(--halloween-color);
+        background-image: linear-gradient(to bottom, var(--halloween-color), var(--halloween-color));
+        border-color: var(--halloween-color);
+    }
+
+    #topBar.navbar>.navbar-inner.navbar-inverse .brand,
+    .navbar-inverse .nav>li>a {
+        color: white;
+    }
+
+    .loader {
+        width: 100%;
+        height: 42px;
+        position: absolute;
+        margin: auto;
+        left: 0;
+        right: 0;
+        top: 0;
+        bottom: 0;
+        text-align: center;
+        line-height: 120px;
+        font-family: sans-serif;
+        color: #ffffff;
+        font-size: 16px;
+        z-index: -1;
+    }
+
+    .loader span {
+        z-index: 3;
+    }
+
+    .loader-bg {
+        position: absolute;
+        left: 0;
+        right: 0;
+        top: 0;
+        bottom: 0;
+        z-index: 2;
+        animation: wave 1s ease-out forwards;
+    }
+
+    .drops {
+        -webkit-filter: url('#liquid');
+        filter: url('#liquid');
+        position: absolute;
+        top: 0;
+        left: 0;
+        bottom: 0;
+        right: 0;
+        z-index: 1;
+        opacity: 0;
+        animation: fade-in .1s linear .4s forwards;
+    }
+
+    .drops1 {
+        left: -44vw;
+    }
+
+    .drops2 {
+        left: -70vw;
+    }
+
+    .drops3 {
+        left: 70vw;
+    }
+
+    .drops1 .drop2 {
+        animation-delay: 0ms;
+    }
+
+    .drops2 .drop2 {
+        animation-delay: 400ms;
+    }
+
+    .drops3 .drop2 {
+        animation-delay: 100ms;
+    }
+
+    .drop1,
+    .drop2 {
+        width: 21px;
+        height: 24px;
+        border-radius: 50%;
+        position: absolute;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        margin: auto;
+        background-color: var(--halloween-color)
+    }
+
+    .drop1 {
+        width: 90px;
+        height: 12px;
+        bottom: 2px;
+        border-radius: 0;
+        z-index: 0;
+    }
+
+    .drop2 {
+        animation: drop 1.3s cubic-bezier(1, .19, .66, .12) .5s infinite;
+    }
+</style>
\ No newline at end of file

From 9962203c5b406f30ebbfce74c77ed8ce8a74cf2e Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 12 Oct 2023 13:36:53 +0200
Subject: [PATCH 61/87] fix: [installer] Kali Linux hack

---
 docs/generic/supportFunctions.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/generic/supportFunctions.md b/docs/generic/supportFunctions.md
index f823b2488c4..47ec7892258 100644
--- a/docs/generic/supportFunctions.md
+++ b/docs/generic/supportFunctions.md
@@ -614,6 +614,14 @@ installRNG () {
 kaliUpgrade () {
   debug "Running various Kali upgrade tasks"
   checkAptLock
+  # Fix Missing keys early
+  sudo wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
+  # /!\ The following is a very ugly dependency hack to make php7.4 work on Kali
+  wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  wget http://ftp.debian.org/debian/pool/main/i/icu/libicu67_67.1-7_amd64.deb
+  sudo dpkg -i libicu67_67.1-7_amd64.deb
+  # EOH End-Of-Hack
   sudo DEBIAN_FRONTEND=noninteractive apt update
   sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y
   sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y

From aee84704ce60e7c7099e2d33ceea7ac53ffb1351 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 12 Oct 2023 13:39:06 +0200
Subject: [PATCH 62/87] chg: [installer] Support Debian 12

---
 INSTALL/INSTALL.tpl.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh
index 0d237b3d1f8..3f3685880ca 100755
--- a/INSTALL/INSTALL.tpl.sh
+++ b/INSTALL/INSTALL.tpl.sh
@@ -844,6 +844,7 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic

From 3541b074f20cb00c16c3c517330c0d7092286080 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 12 Oct 2023 13:42:26 +0200
Subject: [PATCH 63/87] chg: [installer] Update to latest installer

---
 INSTALL/INSTALL.sh        | 9 +++++++++
 INSTALL/INSTALL.sh.sfv    | 6 +++---
 INSTALL/INSTALL.sh.sha1   | 2 +-
 INSTALL/INSTALL.sh.sha256 | 2 +-
 INSTALL/INSTALL.sh.sha384 | 2 +-
 INSTALL/INSTALL.sh.sha512 | 2 +-
 6 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index 6d5e1e2f2f8..4d626255750 100755
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -794,6 +794,14 @@ installRNG () {
 kaliUpgrade () {
   debug "Running various Kali upgrade tasks"
   checkAptLock
+  # Fix Missing keys early
+  sudo wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
+  # /!\ The following is a very ugly dependency hack to make php7.4 work on Kali
+  wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  wget http://ftp.debian.org/debian/pool/main/i/icu/libicu67_67.1-7_amd64.deb
+  sudo dpkg -i libicu67_67.1-7_amd64.deb
+  # EOH End-Of-Hack
   sudo DEBIAN_FRONTEND=noninteractive apt update
   sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y
   sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y
@@ -3597,6 +3605,7 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic
diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv
index 9239dcd0c7f..2d66dabe3bd 100644
--- a/INSTALL/INSTALL.sh.sfv
+++ b/INSTALL/INSTALL.sh.sfv
@@ -1,5 +1,5 @@
-; Generated by RHash v1.4.4 on 2023-08-22 at 17:22.35
+; Generated by RHash v1.4.4 on 2023-10-12 at 13:42.08
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160749  17:22.35 2023-08-22 INSTALL.sh
-INSTALL.sh 06BE6B05BBAD5007BDDDB73DBA2F090A3F4552B1 A4A53EB3EC60FFAD773E8E1D76278315B40042E1B2E62971E73D3F66E9327143 98072442A60BE33F9CCF8C205E4CB2A894CB060566ED9CB835DD4B38C6EDD66B2A94ABE860EFEBD9980EE6C1EF4A5B06 EE56B1BF53930F16CCF13B9C308D55E74D52CF65C1BFB03B890E06476A84F30B2C0AF0F488E34A7A22666B3C1F49866598A35B1EB9F3ADE57427DC56E772B7C9
+;       161244  13:42.07 2023-10-12 INSTALL.sh
+INSTALL.sh 8624B1D834A4C958B16DCE32CEAE88C3D1DC15D7 D7B9E370C85B53BBF7B0F81F5C263B6AB2E534F59B40A6499277F39407FF194A EF5B68E9D0D634C2CADD4FD9B1E5C2A93DBD938F488417F67A2B9C87E8867CB0200293A150CDAEDA369E7FDC476EEC2B 1445710924BC029647CC5AA0EBFFD0A3B6DDAF39D26B5A0E951FFFEEF621677C59470F250ECFB6059C9E200951F98D4F21646F152D6F8931438531F9516A8748
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
index afb8272be77..c35e996e5bd 100644
--- a/INSTALL/INSTALL.sh.sha1
+++ b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-06be6b05bbad5007bdddb73dba2f090a3f4552b1  INSTALL.sh
+8624b1d834a4c958b16dce32ceae88c3d1dc15d7  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
index 711217d8747..c668c2636f2 100644
--- a/INSTALL/INSTALL.sh.sha256
+++ b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-a4a53eb3ec60ffad773e8e1d76278315b40042e1b2e62971e73d3f66e9327143  INSTALL.sh
+d7b9e370c85b53bbf7b0f81f5c263b6ab2e534f59b40a6499277f39407ff194a  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
index cbe9ca7794c..56792531f28 100644
--- a/INSTALL/INSTALL.sh.sha384
+++ b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-98072442a60be33f9ccf8c205e4cb2a894cb060566ed9cb835dd4b38c6edd66b2a94abe860efebd9980ee6c1ef4a5b06  INSTALL.sh
+ef5b68e9d0d634c2cadd4fd9b1e5c2a93dbd938f488417f67a2b9c87e8867cb0200293a150cdaeda369e7fdc476eec2b  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
index 4dc9d26cbdd..ea40a4ad7ce 100644
--- a/INSTALL/INSTALL.sh.sha512
+++ b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-ee56b1bf53930f16ccf13b9c308d55e74d52cf65c1bfb03b890e06476a84f30b2c0af0f488e34a7a22666b3c1f49866598a35b1eb9f3ade57427dc56e772b7c9  INSTALL.sh
+1445710924bc029647cc5aa0ebffd0a3b6ddaf39d26b5a0e951fffeef621677c59470f250ecfb6059c9e200951f98d4f21646f152d6f8931438531f9516a8748  INSTALL.sh

From 4c1ce7ee8e700f2757ac7be9a3bbb59ab6db98cf Mon Sep 17 00:00:00 2001
From: peritz <peritz@pardonmynoot.com>
Date: Thu, 12 Oct 2023 16:05:06 +0100
Subject: [PATCH 64/87] fix: [misp-zmq] Include ZMQ support for connecting to
 Redis over TLS

---
 app/files/scripts/mispzmq/mispzmq.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/files/scripts/mispzmq/mispzmq.py b/app/files/scripts/mispzmq/mispzmq.py
index e2492d8ec2f..4c67684ce71 100644
--- a/app/files/scripts/mispzmq/mispzmq.py
+++ b/app/files/scripts/mispzmq/mispzmq.py
@@ -80,9 +80,14 @@ def _setup(self):
         with open((self.tmp_location / "mispzmq_settings.json").as_posix()) as settings_file:
             self.settings = json.load(settings_file)
         self.namespace = self.settings["redis_namespace"]
-        self.r = redis.StrictRedis(host=self.settings["redis_host"], db=self.settings["redis_database"],
+        # Check if TLS is being used with Redis host
+        redis_host = self.settings["redis_host"]
+        redis_ssl = redis_host.startswith("tls://")
+        if redis_host.startswith("tls://"):
+            redis_host = redis_host[6:]
+        self.r = redis.StrictRedis(host=redis_host, db=self.settings["redis_database"],
                                    password=self.settings["redis_password"], port=self.settings["redis_port"],
-                                   decode_responses=True)
+                                   decode_responses=True, ssl=redis_ssl)
         self.timestamp_settings = time.time()
         self._logger.debug("Connected to Redis {}:{}/{}".format(self.settings["redis_host"], self.settings["redis_port"],
                                                            self.settings["redis_database"]))

From 0cf3921e00f4c6a8b15a24a4ee1b9883e6896228 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Sun, 15 Oct 2023 08:01:21 +0200
Subject: [PATCH 65/87] chg: [misp-galaxy] various updates

---
 app/files/misp-galaxy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index f80bcdd97fd..fe77114b84b 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit f80bcdd97fdf7a841ab57036cb30a314467fa3ba
+Subproject commit fe77114b84b3c40620215fdaf971d825800bb382

From 94585b4dad806efcb62f445d6bbd96b4d7429bc2 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 17 Oct 2023 21:10:12 +0200
Subject: [PATCH 66/87] fix: [warninglists] default to matching types ALL if
 nothing is set

---
 app/Controller/WarninglistsController.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/Controller/WarninglistsController.php b/app/Controller/WarninglistsController.php
index 32c068fa713..a208f0eb515 100644
--- a/app/Controller/WarninglistsController.php
+++ b/app/Controller/WarninglistsController.php
@@ -157,6 +157,9 @@ public function add()
                     unset($warninglist['Warninglist']['entries']);
                     $warninglist['WarninglistEntry'] = $entries;
                 }
+                if (empty($warninglist['Warninglist']['matching_attributes'])) {
+                    $warninglist['Warninglist']['matching_attributes'] = ['ALL'];
+                }
                 if (isset($warninglist['Warninglist']['matching_attributes']) && is_array($warninglist['Warninglist']['matching_attributes'])) {
                     $warninglist['WarninglistType'] = [];
                     foreach ($warninglist['Warninglist']['matching_attributes'] as $attribute) {

From b3591aeefdc71473648271002508b840c4bc70a7 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 17 Oct 2023 21:17:08 +0200
Subject: [PATCH 67/87] new: [workflow-module:publish-event] Added draft of
 module

---
 .../action/Module_publish_event.php           | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 app/Model/WorkflowModules/action/Module_publish_event.php

diff --git a/app/Model/WorkflowModules/action/Module_publish_event.php b/app/Model/WorkflowModules/action/Module_publish_event.php
new file mode 100644
index 00000000000..afc86a7a6fc
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_publish_event.php
@@ -0,0 +1,28 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_webhook.php';
+
+class Module_publish_event extends Module_webhook
+{
+    public $id = 'publish-event';
+    public $name = 'Publish Event';
+    public $version = '0.1';
+    public $description = 'Publish an Event in the context of the workflow';
+    public $icon_path = 'upload';
+
+    private $Event;
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->Event = ClassRegistry::init('Event');
+        $this->params = [
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $result = $this->Event->publish($event_id, null);
+        return $result === true;
+    }
+}

From 2b030fb4961a548b5d54a17dc4a6975141358480 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 17 Oct 2023 22:34:57 +0200
Subject: [PATCH 68/87] fix: [workflow-module:publish-event] Extend correct
 class and use event id

---
 .../WorkflowModules/action/Module_publish_event.php   | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_publish_event.php b/app/Model/WorkflowModules/action/Module_publish_event.php
index afc86a7a6fc..1554677db6c 100644
--- a/app/Model/WorkflowModules/action/Module_publish_event.php
+++ b/app/Model/WorkflowModules/action/Module_publish_event.php
@@ -1,13 +1,16 @@
 <?php
-include_once APP . 'Model/WorkflowModules/action/Module_webhook.php';
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
 
-class Module_publish_event extends Module_webhook
+class Module_publish_event extends WorkflowBaseActionModule
 {
     public $id = 'publish-event';
     public $name = 'Publish Event';
     public $version = '0.1';
     public $description = 'Publish an Event in the context of the workflow';
-    public $icon_path = 'upload';
+    public $icon = 'upload';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $params = [];
 
     private $Event;
 
@@ -22,6 +25,8 @@ public function __construct()
 
     public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
     {
+        $rData = $roamingData->getData();
+        $event_id = $rData['Event']['id'];
         $result = $this->Event->publish($event_id, null);
         return $result === true;
     }

From cc4a51aea486771b96946723046f11cd8fd59f6b Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 18 Oct 2023 16:14:01 +0200
Subject: [PATCH 69/87] fix: [attribute] Log entry with the correct action

---
 app/Model/Attribute.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index ca2c6658ab8..9461b11cf50 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -3204,7 +3204,7 @@ public function logDropped(array $user, array $attribute, $action = 'add')
         $attribute_short = (isset($attribute['category']) ? $attribute['category'] : 'N/A') . '/' . (isset($attribute['type']) ? $attribute['type'] : 'N/A') . ' ' . (isset($attribute['value']) ? $attribute['value'] : 'N/A');
         $eventId = $attribute['event_id'];
         $modelId = $action === 'add' ? 0 : $this->id;
-        $this->loadLog()->createLogEntry($user, 'add', 'Attribute',  $modelId,
+        $this->loadLog()->createLogEntry($user, $action, 'Attribute',  $modelId,
             "Attribute dropped due to validation for Event $eventId failed: $attribute_short",
             'Validation errors: ' . JsonTool::encode($this->validationErrors) . ' Full Attribute: ' . JsonTool::encode($attribute)
         );

From 90cb068a0e401617630f4adf7ade503572ae66aa Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 18 Oct 2023 16:14:52 +0200
Subject: [PATCH 70/87] fix: [sighting:attachOrgToSightings] Stopped double
 unpacking

---
 app/Model/Sighting.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 2562b404d1a..661b48f48e1 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -642,8 +642,8 @@ private function attachOrgToSightings(array $sightings, array $user, $forSync =
                     unset($sighting['org_id']);
                     unset($sighting['Organisation']);
                 } else {
-                    $sighting['org_id'] = $anonOrg['Organisation']['id'];
-                    $sighting['Organisation'] = $anonOrg['Organisation'];
+                    $sighting['org_id'] = $anonOrg['id'];
+                    $sighting['Organisation'] = $anonOrg;
                 }
             }
             $sightings[$k] = $sighting;

From 1a4def8e1636d7b062ee46d358fba6e49bf057fd Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 19 Oct 2023 10:42:51 +0200
Subject: [PATCH 71/87] chg: [eventtimeline:doubleclick] Change the location to
 the object being double-clicked

---
 app/webroot/js/event-timeline.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/webroot/js/event-timeline.js b/app/webroot/js/event-timeline.js
index 2286e456646..37e85ce586c 100644
--- a/app/webroot/js/event-timeline.js
+++ b/app/webroot/js/event-timeline.js
@@ -640,15 +640,16 @@ function edit_item(id, callback) {
     var item = items_timeline.get(id);
     var group = item.group;
     if (group == 'attribute') {
-        simplePopup(baseurl+'/attributes/edit/'+item.orig_id);
+        // openGenericModal(baseurl + '/attributes/edit/' + item.orig_id);
+        // FIXME: Open it up as a popup and make sure edits on fs/ls are correctly saved
+        window.location = baseurl + '/attributes/edit/' + item.orig_id;
     } else if (group == 'object') {
         window.location = baseurl+'/objects/edit/'+item.orig_id;
     }
 }
 
 function handle_doubleClick(data) {
-    // should be replaced by keyboard shortcut: SHIFT+E ?
-    //edit_item(data.item);
+    edit_item(data.item);
 }
 
 function handle_not_seen_enabled(hide, include_hidden) {

From edd937861a20dd025305102816410ca3062c8458 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 19 Oct 2023 11:21:09 +0200
Subject: [PATCH 72/87] chg: [dashboard-widget:worldmap] Added support of
 custom scale in widget config

---
 app/View/Elements/dashboard/Widgets/WorldMap.ctp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/View/Elements/dashboard/Widgets/WorldMap.ctp b/app/View/Elements/dashboard/Widgets/WorldMap.ctp
index 20c4f9a3035..4cf162ee231 100644
--- a/app/View/Elements/dashboard/Widgets/WorldMap.ctp
+++ b/app/View/Elements/dashboard/Widgets/WorldMap.ctp
@@ -12,6 +12,9 @@
         'meta' => 'icon'
     ));
     $randomNumber = rand();
+    if (!empty($config['widget_config']['colour_scale'])) {
+        $data['colour_scale'] = json_encode($config['widget_config']['colour_scale']);
+    }
     if (empty($data['colour_scale'])) {
         $data['colour_scale'] = json_encode(array(
             '#003FBF','#0063BF','#0087BF','#00ACBF','#00BFAD','#00BF89','#00BF64',

From 005609ecd4e7d1bd2f95669aa5c2092c9b3a1bae Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 19 Oct 2023 13:44:36 +0200
Subject: [PATCH 73/87] new: [workflow-trigger:event-before-save] Added trigger

---
 app/Model/Event.php                           | 18 +++++++++++++++
 .../trigger/Module_event_before_save.php      | 22 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 app/Model/WorkflowModules/trigger/Module_event_before_save.php

diff --git a/app/Model/Event.php b/app/Model/Event.php
index fe43472650e..90c608d294c 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -499,6 +499,24 @@ public function beforeSave($options = [])
             $this->data['Event']['uuid'] = CakeText::uuid();
         }
         $this->__beforeSaveData = $this->data['Event'];
+
+        $trigger_id = 'event-before-save';
+        if ($this->isTriggerCallable($trigger_id)) {
+            $event = $this->data;
+            $workflowErrors = [];
+            $logging = [
+                'model' => 'Event',
+                'action' => 'add',
+                'id' => 0,
+                'message' => __('The workflow `%s` prevented the saving of event (%s)', $trigger_id, $event['Event']['uuid']),
+            ];
+            $triggerData = $event;
+            $workflowSuccess = $this->executeTrigger($trigger_id, $triggerData, $workflowErrors, $logging);
+            if (!$workflowSuccess) {
+                return false;
+            }
+        }
+
         return true;
     }
 
diff --git a/app/Model/WorkflowModules/trigger/Module_event_before_save.php b/app/Model/WorkflowModules/trigger/Module_event_before_save.php
new file mode 100644
index 00000000000..aac950e0e17
--- /dev/null
+++ b/app/Model/WorkflowModules/trigger/Module_event_before_save.php
@@ -0,0 +1,22 @@
+<?php
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
+
+class Module_event_before_save extends WorkflowBaseTriggerModule
+{
+    public $id = 'event-before-save';
+    public $scope = 'event';
+    public $name = 'Event Before Save';
+    public $description = 'This trigger is called before an Event or any of its elements is about to be saved in the database';
+    public $icon = 'envelope';
+    public $inputs = 0;
+    public $outputs = 1;
+    public $blocking = true;
+    public $misp_core_format = true;
+    public $trigger_overhead = self::OVERHEAD_HIGH;
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->trigger_overhead_message = __('This trigger is called each time an Event or Attribute is about to be saved. This means that when a large quantity of Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Attributes.');
+    }
+}

From 2a23c1d234e84925223030ba7a02b21af40b1478 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 19 Oct 2023 13:46:14 +0200
Subject: [PATCH 74/87] new: [workflow-modules:add-eventblocklist-entry] Added
 new action module

---
 app/Model/EventBlocklist.php                  | 10 +++
 .../Module_add_eventblocklist_entry.php       | 81 +++++++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 app/Model/WorkflowModules/action/Module_add_eventblocklist_entry.php

diff --git a/app/Model/EventBlocklist.php b/app/Model/EventBlocklist.php
index cd9c0d21f1b..dce41343c73 100644
--- a/app/Model/EventBlocklist.php
+++ b/app/Model/EventBlocklist.php
@@ -79,4 +79,14 @@ public function isBlocked($eventUuid)
     {
         return $this->hasAny(['event_uuid' => $eventUuid]);
     }
+
+    /**
+     * @param string $eventUuid
+     * @return bool
+     */
+    public function addEntry($entry=[])
+    {
+        $this->create();
+        return $this->save($entry);
+    }
 }
diff --git a/app/Model/WorkflowModules/action/Module_add_eventblocklist_entry.php b/app/Model/WorkflowModules/action/Module_add_eventblocklist_entry.php
new file mode 100644
index 00000000000..19fd36c35bb
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_add_eventblocklist_entry.php
@@ -0,0 +1,81 @@
+<?php
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
+
+class Module_add_eventblocklist_entry extends WorkflowBaseActionModule
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'add_eventblocklist_entry';
+    public $name = 'Add Event Blocklist entry';
+    public $description = 'Create a new entry in the Event blocklist table';
+    public $icon = 'ban';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = false;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    private $EventBlocklist;
+    private $Organisation;
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'uuid_hash_path',
+                'label' => 'Event UUID Hash path',
+                'type' => 'hashpath',
+                'placeholder' => 'Event.uuid',
+                'default' => 'Event.uuid',
+                'hashpath' => [
+                    'is_sub_selector' => true
+                ]
+            ],
+            [
+                'id' => 'eventinfo_hash_path',
+                'label' => 'Event Info Hash path',
+                'type' => 'hashpath',
+                'placeholder' => 'Event.info',
+                'default' => 'Event.info',
+                'hashpath' => [
+                    'is_sub_selector' => true
+                ]
+            ],
+            [
+                'id' => 'block_comment',
+                'label' => 'Blocklist Comment',
+                'type' => 'input',
+                'placeholder' => 'Blocked from workflow',
+                'default' => 'Blocked from workflow',
+            ],
+        ];
+
+        $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+        $this->Organisation = ClassRegistry::init('Organisation');
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+
+        $eventUUIDExtractionPath = $params['uuid_hash_path']['value'];
+        $eventUUID = Hash::get($rData, $eventUUIDExtractionPath);
+        $eventInfoExtractionPath = $params['eventinfo_hash_path']['value'];
+        $eventInfo = Hash::get($rData, $eventInfoExtractionPath);
+        $comment = $params['block_comment']['value'];
+
+        $org = $this->Organisation->find('first', ['conditions' => array('Organisation.id' => $rData['Event']['orgc_id']), 'recursive' => -1, 'fields' => ['Organisation.name']]);
+        $entry = [
+            'event_uuid' => $eventUUID,
+            'event_info' => $eventInfo,
+            'event_orgc' => !empty($org['Organisation']['name']) ? $org['Organisation']['name'] : 'unkwown',
+            'comment' => $comment,
+        ];
+        $r = $this->EventBlocklist->addEntry($entry);
+        return true;
+    }
+}

From ad7a0ba01d726d3786ea3dc8b501ea7eb8268f5b Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Thu, 19 Oct 2023 20:07:47 +0200
Subject: [PATCH 75/87] chg: [misp-stix] Bumped latest version

---
 app/View/Helper/UserNameHelper.php | 2 --
 app/files/scripts/misp-stix        | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/app/View/Helper/UserNameHelper.php b/app/View/Helper/UserNameHelper.php
index 18a8ad0ba0c..42a83d79c58 100644
--- a/app/View/Helper/UserNameHelper.php
+++ b/app/View/Helper/UserNameHelper.php
@@ -32,8 +32,6 @@ public function prepend($email)
             return '<i class="fas fa-smile-beam white"></i>&nbsp;';
         } else if (strpos($lower_email, 'rand') !== false && (strpos($lower_email, 'ecrime') !== false)) {
             return '<i class="fas fa-camera white"></i>&nbsp;';
-        } else if ($lower_email === 'christian.studer@circl.lu') {
-            return '<span class="bold white">Mr STIX</span> ';
         } else if ($lower_email === 'sami.mokaddem@circl.lu') {
             return '<span class="bold white">Graphman</span> ';
         } else if (strpos($lower_email, 'm.j.nassette') !== false) {
diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix
index 09e95edf542..88d2e472b0c 160000
--- a/app/files/scripts/misp-stix
+++ b/app/files/scripts/misp-stix
@@ -1 +1 @@
-Subproject commit 09e95edf5426adacba8b45ffe0596c5e8f2c2977
+Subproject commit 88d2e472b0cb41fa6a076be4f5b7e5240bfab285

From ab7dff1c7f3d550a12f2936da62d25ed87d68faa Mon Sep 17 00:00:00 2001
From: Vincenzo Caputo <32276363+vincenzocaputo@users.noreply.github.com>
Date: Sat, 21 Oct 2023 21:05:31 +0000
Subject: [PATCH 76/87] Fix API url in Telegram workflow module

---
 .../WorkflowModules/action/Module_telegram_send_alert.php     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_telegram_send_alert.php b/app/Model/WorkflowModules/action/Module_telegram_send_alert.php
index 241934c7319..9e3e6b46e44 100644
--- a/app/Model/WorkflowModules/action/Module_telegram_send_alert.php
+++ b/app/Model/WorkflowModules/action/Module_telegram_send_alert.php
@@ -51,11 +51,11 @@ public function exec(array $node, WorkflowRoamingData $roamingData, array &$erro
 		'parse_mode' => "HTML",
 	];
 
-	$url = $this->telegram_url . $bot_token . "/sendMessage";
+	$url = $this->telegram_url . "bot" . $bot_token . "/sendMessage";
 
 	$response = $this->doRequest(
 		$url,
-		'json',
+		'application/json',
 		$data
 	);
 

From 2075687b042bd0cdb8eb8d729040769f87acb5ce Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 23 Oct 2023 11:39:06 +0200
Subject: [PATCH 77/87] chg: [event:restSearch] Added support of orgc_id as
 valid filter

---
 app/Model/Event.php | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 90c608d294c..d22dfdc384a 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -186,6 +186,7 @@ class Event extends AppModel
         'order',
         'protected',
         'published',
+        'orgc_id',
     );
 
     public $validate = array(
@@ -1545,6 +1546,7 @@ public function filterEventIds($user, &$params = array(), &$result_count = 0)
                     'event_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                     'publish_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
                     'org' => array('function' => 'set_filter_org', 'pop' => true),
+                    'orgc_id' => array('function' => 'set_filter_orgc_id', 'pop' => true),
                     'uuid' => array('function' => 'set_filter_uuid', 'pop' => true),
                     'published' => array('function' => 'set_filter_published', 'pop' => true),
                     'threat_level_id' => array('function' => 'set_filter_threat_level_id', 'pop' => true),
@@ -1905,6 +1907,9 @@ public function fetchEvent($user, $options = array(), $useCache = false)
         if ($options['published']) {
             $conditions['AND'][] = array('Event.published' => $options['published']);
         }
+        if ($options['orgc_id']) {
+            $conditions['AND'][] = array('Event.orgc_id' => $options['orgc_id']);
+        }
         if (!empty($options['includeRelatedTags'])) {
             $options['includeGranularCorrelations'] = 1;
         }
@@ -2639,6 +2644,15 @@ public function set_filter_sharing_group(&$params, $conditions, $options)
         return $conditions;
     }
 
+    public function set_filter_orgc_id(&$params, $conditions, $options)
+    {
+        if (!empty($params['orgc_id'])) {
+            $orgFilter = ['OR' => $params['orgc_id']];
+            $conditions = $this->generic_add_filter($conditions, $orgFilter, 'Event.orgc_id');
+        }
+        return $conditions;
+    }
+
     public function set_filter_org(&$params, $conditions, $options)
     {
         if (!empty($params['org'])) {

From c952fcad4bf4e3ecacdd7cf66168ca37bc3ea984 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 24 Oct 2023 09:26:41 +0200
Subject: [PATCH 78/87] fix: [objects:edit] Restored behavior of upgrading
 object to newer template

---
 app/Controller/ObjectsController.php | 5 +++--
 app/Model/MispObject.php             | 2 +-
 app/View/Objects/add.ctp             | 3 +++
 app/View/Objects/revise_object.ctp   | 3 +++
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/app/Controller/ObjectsController.php b/app/Controller/ObjectsController.php
index c58aca67151..fdceb7f5fde 100644
--- a/app/Controller/ObjectsController.php
+++ b/app/Controller/ObjectsController.php
@@ -27,7 +27,7 @@ public function beforeFilter()
         }
     }
 
-    public function revise_object($action, $event_id, $template_id, $object_id = false, $similar_objects_display_threshold=15)
+    public function revise_object($action, $event_id, $template_id, $object_id = false, $update_template_available = false, $similar_objects_display_threshold=15)
     {
         if (!$this->request->is('post') && !$this->request->is('put')) {
             throw new MethodNotAllowedException(__('This action can only be reached via POST requests'));
@@ -79,6 +79,7 @@ public function revise_object($action, $event_id, $template_id, $object_id = fal
         $this->set('object_id', $object_id);
         $this->set('event', $event);
         $this->set('data', $this->request->data);
+        $this->set('update_template_available', !empty($update_template_available));
         // Make sure the data stored in the session applies to this object. User might be prompted to perform a merge with another object if the session's data is somehow not cleaned
         $curObjectTmpUuid = CakeText::uuid();
         $this->set('cur_object_tmp_uuid', $curObjectTmpUuid);
@@ -368,7 +369,7 @@ public function edit($id, $update_template_available=false, $onlyAddNewAttribute
             $this->Flash->error('Object cannot be edited, no valid template found. ', ['params' => ['url' => sprintf('/objects/edit/%s/1/0', $id), 'urlName' => __('Force update anyway')]]);
             $this->redirect(array('controller' => 'events', 'action' => 'view', $object['Object']['event_id']));
         }
-        if (!empty($template)) {
+        if (!empty($template) || $update_template_available) {
             $templateData = $this->MispObject->resolveUpdatedTemplate($template, $object, $update_template_available);
             $this->set('updateable_attribute', $templateData['updateable_attribute']);
             $this->set('not_updateable_attribute', $templateData['not_updateable_attribute']);
diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php
index 490cd45bacf..44f20aecc98 100644
--- a/app/Model/MispObject.php
+++ b/app/Model/MispObject.php
@@ -1398,7 +1398,7 @@ public function resolveUpdatedTemplate($template, $object, $update_template_avai
         ));
         $template_difference = array();
         if (!empty($newer_template)) {
-            $toReturn['newer_template_version'] = !$newer_template['ObjectTemplate']['version'];
+            $toReturn['newer_template_version'] = $newer_template['ObjectTemplate']['version'];
             $newer_template_temp = Hash::remove(Hash::remove($newer_template['ObjectTemplateElement'], '{n}.id'), '{n}.object_template_id');
             if (!empty($template)) {
                 // ignore IDs for comparison
diff --git a/app/View/Objects/add.ctp b/app/View/Objects/add.ctp
index d10a9fe1fe5..9f66bc7a575 100644
--- a/app/View/Objects/add.ctp
+++ b/app/View/Objects/add.ctp
@@ -5,6 +5,9 @@
         $url = $baseurl . '/objects/revise_object/add/' . $event['Event']['id'] . '/' . $template['ObjectTemplate']['id'];
     } else {
         $url = $baseurl . '/objects/revise_object/edit/' . $event['Event']['id'] . '/' . $template['ObjectTemplate']['id'] . '/' . h($object['Object']['id']);
+        if ($update_template_available) {
+            $url .= '/1';
+        }
     }
     echo $this->Form->create('Object', array('id', 'url' => $url, 'enctype' => 'multipart/form-data'));
 ?>
diff --git a/app/View/Objects/revise_object.ctp b/app/View/Objects/revise_object.ctp
index e08f1209417..4218ed859fd 100644
--- a/app/View/Objects/revise_object.ctp
+++ b/app/View/Objects/revise_object.ctp
@@ -22,6 +22,9 @@ $tableData = [
         $url = $baseurl . '/objects/add/' . $event['Event']['id'] . '/' . $template['ObjectTemplate']['id'];
     } else {
         $url = $baseurl . '/objects/edit/' . $object_id;
+        if (!empty($update_template_available)) {
+            $url .= '/1';
+        }
     }
     echo $this->Form->create('Object', array('id', 'url' => $url));
     $formSettings = array(

From 8407961025f89b397e0d137d73e99605a4f04dc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 24 Oct 2023 15:10:57 +0200
Subject: [PATCH 79/87] chg: [PyMISP] Bump version

---
 PyMISP                           | 2 +-
 app/Controller/AppController.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/PyMISP b/PyMISP
index 2dff8fb9b8d..5d39d81152b 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 2dff8fb9b8d8a5214e5a364da40ca6afca56f09d
+Subproject commit 5d39d81152b386ca190c09cb2c9347ba2de77384
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index c4a3512c98c..d2eaf1dd34d 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '155';
-    public $pyMispVersion = '2.4.176';
+    public $pyMispVersion = '2.4.178';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';

From 1b952c143afd2b48f7a348bb77e92e4fe2571879 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 24 Oct 2023 15:46:34 +0200
Subject: [PATCH 80/87] fix: [PyMISP] missing changes in testsuite

---
 PyMISP | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PyMISP b/PyMISP
index 5d39d81152b..8318bce57ed 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 5d39d81152b386ca190c09cb2c9347ba2de77384
+Subproject commit 8318bce57ed8bb6bc999459567d21540452a3922

From 5f1a3b1df2aad578d9ebd3116330b750c368aa64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Tue, 24 Oct 2023 16:02:21 +0200
Subject: [PATCH 81/87] fix: [PyMISP] Another fix in tests.

---
 PyMISP | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PyMISP b/PyMISP
index 8318bce57ed..f38c56cf6dd 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 8318bce57ed8bb6bc999459567d21540452a3922
+Subproject commit f38c56cf6dd02423373700441815e200a9e8e280

From f2ff8441e3f736ee2d2d543b4a22a37adadf87fd Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 24 Oct 2023 19:55:36 +0200
Subject: [PATCH 82/87] fix: [user search] in index, removed old style authkey
 as a valid search field

---
 app/Controller/UsersController.php | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 42852dfc95f..a59ad5bfb9d 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -419,8 +419,7 @@ public function admin_index()
                                         'OR' => array(
                                             'UPPER(User.email) LIKE' => $searchValue,
                                             'UPPER(Organisation.name) LIKE' => $searchValue,
-                                            'UPPER(Role.name) LIKE' => $searchValue,
-                                            'UPPER(User.authkey) LIKE' => $searchValue,
+                                            'UPPER(Role.name) LIKE' => $searchValue
                                         ),
                                     );
                                 } else {

From 15e8f29d5b14f788a48c57a1827a50a7e17a3d6e Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 24 Oct 2023 22:32:56 +0200
Subject: [PATCH 83/87] chg: [submodule] update

---
 app/files/misp-galaxy  | 2 +-
 app/files/misp-objects | 2 +-
 app/files/taxonomies   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index fe77114b84b..c585caa4db3 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit fe77114b84b3c40620215fdaf971d825800bb382
+Subproject commit c585caa4db3ea6dfb9fd5ba2099a9b648d3ef937
diff --git a/app/files/misp-objects b/app/files/misp-objects
index 3c2b62d3c39..5feb0527321 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit 3c2b62d3c390fbe9a28be417422317a8af0a22a4
+Subproject commit 5feb0527321ecfc5a7028df5db561c95d0fb4798
diff --git a/app/files/taxonomies b/app/files/taxonomies
index 8d8433399f0..e8892b6cf91 160000
--- a/app/files/taxonomies
+++ b/app/files/taxonomies
@@ -1 +1 @@
-Subproject commit 8d8433399f0d00651238237be091fe63768a924f
+Subproject commit e8892b6cf91551d93acf94ce52a36a7112e756cc

From 45e5b3d15732a9e6b19a7ae09426dda092aff2a3 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 24 Oct 2023 22:37:50 +0200
Subject: [PATCH 84/87] chg: [version] bump

---
 VERSION.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION.json b/VERSION.json
index 61f654926a0..264949bc5e1 100644
--- a/VERSION.json
+++ b/VERSION.json
@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":177}
+{"major":2, "minor":4, "hotfix":178}

From 731eed55e90f73b1c338ed323fd99df21e2f4aa1 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 31 Oct 2023 08:18:26 +0100
Subject: [PATCH 85/87] fix: [ui:global_menu] Make sure right_menu is defined

---
 app/View/Elements/global_menu.ctp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp
index a11e37159b2..4a8e50968cb 100755
--- a/app/View/Elements/global_menu.ctp
+++ b/app/View/Elements/global_menu.ctp
@@ -550,7 +550,9 @@
                            <i class="fas fa-cat fa-stack-1x fa-flip-horizontal" style="bottom: 8px; left: 2px;"></i>
                        </span>'
         ];
-        $menu_right = array_merge([$tmp], $menu_right);
+        if (isset($menu_right)) {
+            $menu_right = array_merge([$tmp], $menu_right);
+        }
     }
 ?>
 <div id="topBar" class="navbar navbar-inverse <?= isset($debugMode) ? $debugMode : 'debugOff' ?>" style="z-index:100;">

From de8075a115cf91188ab3bcfb283f1d61fe7f88e6 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Tue, 31 Oct 2023 14:50:23 +0100
Subject: [PATCH 86/87] chg: [requirements] PyMISP version updated

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 90a0bd762eb..42b1b5bf2e5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1
 mixbox>=1.0.5
 plyara>=2.1.1
 pydeep2>=0.5.1
-pymisp==2.4.176
+pymisp==2.4.178
 python-magic>=0.4.27
 pyzmq>=25.1.1
 redis>=5.0.1

From a0a2e57fbda46f9d27ca1498c4c036e176ae74e4 Mon Sep 17 00:00:00 2001
From: Andras Iklody <andras.iklody@gmail.com>
Date: Wed, 1 Nov 2023 06:14:14 +0100
Subject: [PATCH 87/87] fix: [UsernameHelper] resolved confusion

Based on etymological discoveries, this long standing issue has been resolved.
---
 app/View/Helper/UserNameHelper.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/View/Helper/UserNameHelper.php b/app/View/Helper/UserNameHelper.php
index 42a83d79c58..972eee7f9b0 100644
--- a/app/View/Helper/UserNameHelper.php
+++ b/app/View/Helper/UserNameHelper.php
@@ -25,7 +25,7 @@ public function prepend($email)
             (strpos($lower_email, 'saad') !== false && strpos($lower_email, 'thehive-project')) ||
             strpos($lower_email, 'saad.kadhi') !== false
         ) {
-            return '<i class="fas fa-frown white"></i>&nbsp;';
+            return '<i class="fas fa-smile white"></i>&nbsp;';
         } else if (strpos($lower_email, 'enrico.lovat') !== false) {
             return '<i class="fas fa-horse-head white"></i>&nbsp;';
         } else if (strpos($lower_email, 'christophe.vandeplas') !== false) {