From bdc8af0efb28203d7d7b147e1196601a2472eaeb Mon Sep 17 00:00:00 2001
From: jean-baptiste-perez-bib <jean-baptiste.perez_ext@michelin.com>
Date: Thu, 5 Dec 2024 15:51:30 +0100
Subject: [PATCH 1/4] Allow showing processing timeline events (backend)

The POST /sketches/{sketchId}/explore/ endpoint can now be parameterized
to include processing timeline indices.
---
 timesketch/api/v1/resources/aggregation.py |  2 +-
 timesketch/api/v1/resources/explore.py     |  8 ++++++--
 timesketch/lib/utils.py                    | 10 ++++++++--
 timesketch/models/sketch.py                |  4 ++--
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/timesketch/api/v1/resources/aggregation.py b/timesketch/api/v1/resources/aggregation.py
index 1b25827e34..de3d13064a 100644
--- a/timesketch/api/v1/resources/aggregation.py
+++ b/timesketch/api/v1/resources/aggregation.py
@@ -468,7 +468,7 @@ def post(self, sketch_id):
         sketch_indices = {
             t.searchindex.index_name
             for t in sketch.timelines
-            if t.get_status.status.lower() == "ready"
+            if t.get_status.status.lower() in ["ready", "processing"]
         }
 
         aggregation_dsl = form.aggregation_dsl.data
diff --git a/timesketch/api/v1/resources/explore.py b/timesketch/api/v1/resources/explore.py
index c45a917f87..f2b38a4149 100644
--- a/timesketch/api/v1/resources/explore.py
+++ b/timesketch/api/v1/resources/explore.py
@@ -142,7 +142,9 @@ def post(self, sketch_id):
         query_filter = request.json.get("filter", {})
         parent = request.json.get("parent", None)
         incognito = request.json.get("incognito", False)
-
+        include_processing_timelines = request.json.get(
+            "includeProcessingTimelines", False
+        )
         return_field_string = form.fields.data
         if return_field_string:
             return_fields = [x.strip() for x in return_field_string.split(",")]
@@ -163,7 +165,9 @@ def post(self, sketch_id):
 
         # Make sure that the indices in the filter are part of the sketch.
         # This will also remove any deleted timeline from the search result.
-        indices, timeline_ids = get_validated_indices(indices, sketch)
+        indices, timeline_ids = get_validated_indices(
+            indices, sketch, include_processing_timelines
+        )
 
         # Remove indices that don't exist from search.
         indices = utils.validate_indices(indices, self.datastore)
diff --git a/timesketch/lib/utils.py b/timesketch/lib/utils.py
index 43bb552766..9cb39c091b 100644
--- a/timesketch/lib/utils.py
+++ b/timesketch/lib/utils.py
@@ -526,21 +526,27 @@ def read_and_validate_jsonl(
             )
 
 
-def get_validated_indices(indices, sketch):
+def get_validated_indices(indices, sketch, include_processing_timelines=False):
     """Exclude any deleted search index references.
 
     Args:
         indices: List of indices from the user
         sketch: A sketch object (instance of models.sketch.Sketch).
+        include_processing_timelines: True to include processing
+          timelines, Flase otherwise.
 
     Returns:
         Tuple of two items:
           List of indices with those removed that is not in the sketch
           List of timeline IDs that should be part of the output.
     """
+    allowed_statuses = ["ready"]
+    if include_processing_timelines:
+        allowed_statuses.append("processing")
+
     sketch_structure = {}
     for timeline in sketch.timelines:
-        if timeline.get_status.status.lower() != "ready":
+        if timeline.get_status.status.lower() not in allowed_statuses:
             continue
         index_ = timeline.searchindex.index_name
         sketch_structure.setdefault(index_, [])
diff --git a/timesketch/models/sketch.py b/timesketch/models/sketch.py
index cda99d7bf0..f41350ca06 100644
--- a/timesketch/models/sketch.py
+++ b/timesketch/models/sketch.py
@@ -135,7 +135,7 @@ def get_view_urls(self):
 
     @property
     def active_timelines(self):
-        """List timelines that are ready for analysis.
+        """List timelines that are being processed or ready for analysis.
 
         Returns:
             List of instances of timesketch.models.sketch.Timeline
@@ -144,7 +144,7 @@ def active_timelines(self):
         for timeline in self.timelines:
             timeline_status = timeline.get_status.status
             index_status = timeline.searchindex.get_status.status
-            if (timeline_status or index_status) in ("processing", "fail", "archived"):
+            if (timeline_status or index_status) in ("fail", "archived"):
                 continue
             _timelines.append(timeline)
         return _timelines

From 28045746be6e8ded0e9a6280d12516f1b021a265 Mon Sep 17 00:00:00 2001
From: jean-baptiste-perez-bib <jean-baptiste.perez_ext@michelin.com>
Date: Mon, 23 Dec 2024 15:51:29 +0100
Subject: [PATCH 2/4] Add a user setting to enable processing timeline events

Default value is "false" (legacy behaviour).
---
 .../frontend-ng/src/components/SettingsDialog.vue   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/timesketch/frontend-ng/src/components/SettingsDialog.vue b/timesketch/frontend-ng/src/components/SettingsDialog.vue
index ca7e789dd2..719fd313a9 100644
--- a/timesketch/frontend-ng/src/components/SettingsDialog.vue
+++ b/timesketch/frontend-ng/src/components/SettingsDialog.vue
@@ -40,6 +40,17 @@ limitations under the License.
           >
         </v-list-item-content>
       </v-list-item>
+      <v-list-item>
+        <v-list-item-action>
+          <v-switch v-model="settings.showProcessingTimelineEvents" color="primary" @change="saveSettings()"></v-switch>
+        </v-list-item-action>
+        <v-list-item-content>
+          <v-list-item-title>Show processing timeline events</v-list-item-title>
+          <v-list-item-subtitle
+          >Select to include events from timelines in <strong>processing</strong> state</v-list-item-subtitle
+          >
+        </v-list-item-content>
+      </v-list-item>
     </v-list>
   </v-card>
 </template>
@@ -49,6 +60,7 @@ import ApiClient from '../utils/RestApiClient'
 
 const DEFAULT_SETTINGS = {
   showLeftPanel: true,
+  showProcessingTimelineEvents: false,
 }
 
 export default {
@@ -57,6 +69,7 @@ export default {
       settings: {
         showLeftPanel: true,
         generateQuery: true,
+        showProcessingTimelineEvents: false,
       },
     }
   },

From 86f6b7ef7cbf56795c2feeb664db5676c1832d31 Mon Sep 17 00:00:00 2001
From: jean-baptiste-perez-bib <jean-baptiste.perez_ext@michelin.com>
Date: Mon, 23 Dec 2024 14:26:06 +0100
Subject: [PATCH 3/4] Allow showing processing timeline events (frontend)

---
 .../src/components/Explore/EventList.vue      |  5 +++++
 .../src/components/Explore/TimelineChip.vue   |  4 ++--
 .../components/Explore/TimelineComponent.vue  | 14 ++++++++----
 .../components/LeftPanel/TimelinesTable.vue   | 22 +++++++++++++++----
 4 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/timesketch/frontend-ng/src/components/Explore/EventList.vue b/timesketch/frontend-ng/src/components/Explore/EventList.vue
index 53918775e7..caa5824be9 100644
--- a/timesketch/frontend-ng/src/components/Explore/EventList.vue
+++ b/timesketch/frontend-ng/src/components/Explore/EventList.vue
@@ -681,6 +681,9 @@ export default {
     activeContext() {
       return this.$store.state.activeContext
     },
+    settings() {
+      return this.$store.state.settings
+    },
     filterChips: function () {
       return this.currentQueryFilter.chips.filter((chip) => chip.type === 'label' || chip.type === 'term')
     },
@@ -864,6 +867,8 @@ export default {
       formData['facet'] = this.activeContext.facetId
       formData['question'] = this.activeContext.questionId
 
+      formData['includeProcessingTimelines'] = this.settings.showProcessingTimelineEvents
+
       ApiClient.search(this.sketch.id, formData)
         .then((response) => {
           this.eventList.objects = response.data.objects
diff --git a/timesketch/frontend-ng/src/components/Explore/TimelineChip.vue b/timesketch/frontend-ng/src/components/Explore/TimelineChip.vue
index aeafed1ce7..e99a444ca8 100644
--- a/timesketch/frontend-ng/src/components/Explore/TimelineChip.vue
+++ b/timesketch/frontend-ng/src/components/Explore/TimelineChip.vue
@@ -50,7 +50,7 @@ limitations under the License.
             <template v-slot:activator="{ on: onTooltip, attrs }">
               <span
                 class="timeline-name-ellipsis"
-                :class="{ disabled: !isSelected && slotProps.timelineStatus === 'ready' }"
+                :class="{ disabled: !isSelected && (slotProps.timelineStatus === 'processing' || slotProps.timelineStatus === 'ready') }"
                 v-bind="attrs"
                 v-on="onTooltip"
                 >{{ timeline.name }}</span
@@ -88,7 +88,7 @@ export default {
   },
   methods: {
     timelineStyle(timelineStatus) {
-      const greyOut = timelineStatus === 'ready' && !this.isSelected
+      const greyOut = (timelineStatus === 'processing' || timelineStatus === 'ready') && !this.isSelected
       return {
         opacity: greyOut ? '50%' : '100%',
         backgroundColor: this.$vuetify.theme.dark ? '#303030' : '#f5f5f5',
diff --git a/timesketch/frontend-ng/src/components/Explore/TimelineComponent.vue b/timesketch/frontend-ng/src/components/Explore/TimelineComponent.vue
index 49b3d37411..b264dd1b89 100644
--- a/timesketch/frontend-ng/src/components/Explore/TimelineComponent.vue
+++ b/timesketch/frontend-ng/src/components/Explore/TimelineComponent.vue
@@ -20,7 +20,7 @@ limitations under the License.
 -->
 <template>
   <span>
-    <v-dialog v-if="timelineStatus === 'processing'" v-model="dialogStatus" width="600">
+    <v-dialog v-if="!timelineAvailable" v-model="dialogStatus" width="600">
       <template v-slot:activator="{ on, attrs }">
         <slot
           name="processing"
@@ -155,7 +155,7 @@ limitations under the License.
             </v-card>
           </v-dialog>
 
-          <v-list-item v-if="timelineStatus === 'ready'" @click="$emit('toggle', timeline)">
+          <v-list-item v-if="timelineAvailable" @click="$emit('toggle', timeline)">
             <v-list-item-action>
               <v-icon v-if="isSelected">mdi-eye-off</v-icon>
               <v-icon v-else>mdi-eye</v-icon>
@@ -164,7 +164,7 @@ limitations under the License.
             <v-list-item-subtitle v-else>Re-enable</v-list-item-subtitle>
           </v-list-item>
 
-          <v-list-item v-if="timelineStatus === 'ready'" @click="$emit('disableAllOtherTimelines', timeline)">
+          <v-list-item v-if="timelineAvailable" @click="$emit('disableAllOtherTimelines', timeline)">
             <v-list-item-action>
               <v-icon>mdi-checkbox-marked-circle-minus-outline</v-icon>
             </v-list-item-action>
@@ -245,7 +245,7 @@ limitations under the License.
             <v-list-item-subtitle>Run Analyzers</v-list-item-subtitle>
           </v-list-item>
 
-          <v-list-item style="cursor: pointer" @click="deleteConfirmation = true">
+          <v-list-item v-if="timelineAvailable" style="cursor: pointer" @click="deleteConfirmation = true">
             <v-list-item-action>
               <v-icon>mdi-trash-can-outline</v-icon>
             </v-list-item-action>
@@ -399,12 +399,18 @@ export default {
     timelineFailed() {
       return this.timelineStatus === 'fail'
     },
+    timelineAvailable() {
+      return this.timelineStatus === 'ready' || (this.showProcessingTimelineEvents && this.timelineStatus === 'processing')
+    },
     timelineChipColor() {
       if (!this.timeline.color.startsWith('#')) {
         return '#' + this.timeline.color
       }
       return this.timeline.color
     },
+    showProcessingTimelineEvents() {
+      return this.$store.state.settings.showProcessingTimelineEvents
+    },
   },
   methods: {
     openDialog() {
diff --git a/timesketch/frontend-ng/src/components/LeftPanel/TimelinesTable.vue b/timesketch/frontend-ng/src/components/LeftPanel/TimelinesTable.vue
index 1f21eecf0a..b78b9cf7e1 100644
--- a/timesketch/frontend-ng/src/components/LeftPanel/TimelinesTable.vue
+++ b/timesketch/frontend-ng/src/components/LeftPanel/TimelinesTable.vue
@@ -195,6 +195,14 @@ export default {
         return a.name.localeCompare(b.name)
       })
     },
+    settings() {
+      return this.$store.state.settings
+    },
+  },
+  watch: {
+    'settings.showProcessingTimelineEvents': function () {
+      this.updateEnabledTimelines()
+    },
   },
   methods: {
     isEnabled(timeline) {
@@ -206,6 +214,15 @@ export default {
     disableAllOtherTimelines(timeline) {
       this.$store.dispatch('updateEnabledTimelines', [timeline.id])
     },
+    updateEnabledTimelines() {
+      this.$store.dispatch(
+        'updateEnabledTimelines',
+        this.activeTimelines
+          .filter((tl) => this.$store.state.enabledTimelines.includes(tl.id))
+          .filter((tl) => this.settings.showProcessingTimelineEvents || tl.status[0].status !== 'processing')
+          .map((tl) => tl.id)
+      )
+    },
     timelineStyle(timelineStatus, isSelected) {
       const greyOut = timelineStatus === 'ready' && !isSelected
       return {
@@ -237,10 +254,7 @@ export default {
     }
   },
   created() {
-    this.$store.dispatch(
-      'updateEnabledTimelines',
-      this.activeTimelines.map((tl) => tl.id)
-    )
+    this.updateEnabledTimelines()
   },
   mounted() {
     EventBus.$on('updateCountPerTimeline', this.updateCountPerTimeline)

From a3d287e1989e2da7d07a77a3e4fa207a096e1f81 Mon Sep 17 00:00:00 2001
From: jean-baptiste-perez-bib <jean-baptiste.perez_ext@michelin.com>
Date: Wed, 8 Jan 2025 14:51:34 +0100
Subject: [PATCH 4/4] Add a banner when displaying processing timeline events

---
 .../src/components/Explore/EventList.vue      | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/timesketch/frontend-ng/src/components/Explore/EventList.vue b/timesketch/frontend-ng/src/components/Explore/EventList.vue
index caa5824be9..dd993ce079 100644
--- a/timesketch/frontend-ng/src/components/Explore/EventList.vue
+++ b/timesketch/frontend-ng/src/components/Explore/EventList.vue
@@ -15,6 +15,15 @@ limitations under the License.
 -->
 <template>
   <div>
+    <v-alert
+      v-model="showBanner"
+      dense
+      dismissible
+      type="info"
+    >
+      Processing timelines can be enabled but their events may be incomplete.
+    </v-alert>
+
     <v-dialog v-model="exportDialog" width="700">
       <v-card flat class="pa-5">
         <v-progress-circular indeterminate size="20" width="1"></v-progress-circular>
@@ -581,6 +590,7 @@ export default {
       showHistogram: false,
       branchParent: null,
       sortOrderAsc: true,
+      showBanner: false,
     }
   },
   computed: {
@@ -873,6 +883,7 @@ export default {
         .then((response) => {
           this.eventList.objects = response.data.objects
           this.eventList.meta = response.data.meta
+          this.updateShowBanner()
           this.searchInProgress = false
           EventBus.$emit('updateCountPerTimeline', response.data.meta.count_per_timeline)
           this.$emit('countPerTimeline', response.data.meta.count_per_timeline)
@@ -1026,6 +1037,15 @@ export default {
         })
         .catch((e) => {})
     },
+    updateShowBanner: function() {
+      // Show banner only when processing timelines are enabled and at
+      // least one enabled timeline is the "processing" state.
+      this.showBanner =
+        this.settings.showProcessingTimelineEvents &&
+        this.$store.state.sketch.active_timelines
+          .filter(tl => this.$store.state.enabledTimelines.includes(tl.id))
+          .some(tl => tl.status && tl.status[0].status === 'processing')
+    },
   },
   watch: {
     tableOptions: {
@@ -1065,6 +1085,11 @@ export default {
       },
       deep: true,
     },
+    'settings.showProcessingTimelineEvents': {
+      handler() {
+        this.updateShowBanner()
+      },
+    },
   },
   created() {
     if (Object.keys(this.queryRequest).length) {