Get processing timeline events

timesketch/api/v1/resources/aggregation.py

@@ -468,7 +468,7 @@ def post(self, sketch_id):
         sketch_indices = {
             t.searchindex.index_name
             for t in sketch.timelines
-            if t.get_status.status.lower() == "ready"
+            if t.get_status.status.lower() in ["ready", "processing"]
         }
 
         aggregation_dsl = form.aggregation_dsl.data

timesketch/api/v1/resources/explore.py

@@ -142,7 +142,9 @@ def post(self, sketch_id):
         query_filter = request.json.get("filter", {})
         parent = request.json.get("parent", None)
         incognito = request.json.get("incognito", False)
-
+        include_processing_timelines = request.json.get(
+            "includeProcessingTimelines", False
+        )
         return_field_string = form.fields.data
         if return_field_string:
             return_fields = [x.strip() for x in return_field_string.split(",")]
@@ -163,7 +165,9 @@ def post(self, sketch_id):
 
         # Make sure that the indices in the filter are part of the sketch.
         # This will also remove any deleted timeline from the search result.
-        indices, timeline_ids = get_validated_indices(indices, sketch)
+        indices, timeline_ids = get_validated_indices(
+            indices, sketch, include_processing_timelines
+        )
 
         # Remove indices that don't exist from search.
         indices = utils.validate_indices(indices, self.datastore)

timesketch/frontend-ng/src/components/Explore/EventList.vue

@@ -15,6 +15,15 @@ limitations under the License.
 -->
 <template>
   <div>
+    <v-alert
+      v-model="showBanner"
+      dense
+      dismissible
+      type="info"
+    >
+      Processing timelines can be enabled but their events may be incomplete.
+    </v-alert>
+
     <v-dialog v-model="exportDialog" width="700">
       <v-card flat class="pa-5">
         <v-progress-circular indeterminate size="20" width="1"></v-progress-circular>
@@ -581,6 +590,7 @@ export default {
       showHistogram: false,
       branchParent: null,
       sortOrderAsc: true,
+      showBanner: false,
     }
   },
   computed: {
@@ -681,6 +691,9 @@ export default {
     activeContext() {
       return this.$store.state.activeContext
     },
+    settings() {
+      return this.$store.state.settings
+    },
     filterChips: function () {
       return this.currentQueryFilter.chips.filter((chip) => chip.type === 'label' || chip.type === 'term')
     },
@@ -864,10 +877,13 @@ export default {
       formData['facet'] = this.activeContext.facetId
       formData['question'] = this.activeContext.questionId
 
+      formData['includeProcessingTimelines'] = this.settings.showProcessingTimelineEvents
+
       ApiClient.search(this.sketch.id, formData)
         .then((response) => {
           this.eventList.objects = response.data.objects
           this.eventList.meta = response.data.meta
+          this.updateShowBanner()
           this.searchInProgress = false
           EventBus.$emit('updateCountPerTimeline', response.data.meta.count_per_timeline)
           this.$emit('countPerTimeline', response.data.meta.count_per_timeline)
@@ -1021,6 +1037,15 @@ export default {
         })
         .catch((e) => {})
     },
+    updateShowBanner: function() {
+      // Show banner only when processing timelines are enabled and at
+      // least one enabled timeline is the "processing" state.
+      this.showBanner =
+        this.settings.showProcessingTimelineEvents &&
+        this.$store.state.sketch.active_timelines
+          .filter(tl => this.$store.state.enabledTimelines.includes(tl.id))
+          .some(tl => tl.status && tl.status[0].status === 'processing')
+    },
   },
   watch: {
     tableOptions: {
@@ -1060,6 +1085,11 @@ export default {
       },
       deep: true,
     },
+    'settings.showProcessingTimelineEvents': {
+      handler() {
+        this.updateShowBanner()
+      },
+    },
   },
   created() {
     if (Object.keys(this.queryRequest).length) {

timesketch/frontend-ng/src/components/Explore/TimelineChip.vue

@@ -50,7 +50,7 @@ limitations under the License.
             <template v-slot:activator="{ on: onTooltip, attrs }">
               <span
                 class="timeline-name-ellipsis"
-                :class="{ disabled: !isSelected && slotProps.timelineStatus === 'ready' }"
+                :class="{ disabled: !isSelected && (slotProps.timelineStatus === 'processing' || slotProps.timelineStatus === 'ready') }"
                 v-bind="attrs"
                 v-on="onTooltip"
                 >{{ timeline.name }}</span
@@ -88,7 +88,7 @@ export default {
   },
   methods: {
     timelineStyle(timelineStatus) {
-      const greyOut = timelineStatus === 'ready' && !this.isSelected
+      const greyOut = (timelineStatus === 'processing' || timelineStatus === 'ready') && !this.isSelected
       return {
         opacity: greyOut ? '50%' : '100%',
         backgroundColor: this.$vuetify.theme.dark ? '#303030' : '#f5f5f5',

timesketch/frontend-ng/src/components/Explore/TimelineComponent.vue

@@ -20,7 +20,7 @@ limitations under the License.
 -->
 <template>
   <span>
-    <v-dialog v-if="timelineStatus === 'processing'" v-model="dialogStatus" width="600">
+    <v-dialog v-if="!timelineAvailable" v-model="dialogStatus" width="600">
       <template v-slot:activator="{ on, attrs }">
         <slot
           name="processing"
@@ -155,7 +155,7 @@ limitations under the License.
             </v-card>
           </v-dialog>
 
-          <v-list-item v-if="timelineStatus === 'ready'" @click="$emit('toggle', timeline)">
+          <v-list-item v-if="timelineAvailable" @click="$emit('toggle', timeline)">
             <v-list-item-action>
               <v-icon v-if="isSelected">mdi-eye-off</v-icon>
               <v-icon v-else>mdi-eye</v-icon>
@@ -164,7 +164,7 @@ limitations under the License.
             <v-list-item-subtitle v-else>Re-enable</v-list-item-subtitle>
           </v-list-item>
 
-          <v-list-item v-if="timelineStatus === 'ready'" @click="$emit('disableAllOtherTimelines', timeline)">
+          <v-list-item v-if="timelineAvailable" @click="$emit('disableAllOtherTimelines', timeline)">
             <v-list-item-action>
               <v-icon>mdi-checkbox-marked-circle-minus-outline</v-icon>
             </v-list-item-action>
@@ -245,7 +245,7 @@ limitations under the License.
             <v-list-item-subtitle>Run Analyzers</v-list-item-subtitle>
           </v-list-item>
 
-          <v-list-item style="cursor: pointer" @click="deleteConfirmation = true">
+          <v-list-item v-if="timelineAvailable" style="cursor: pointer" @click="deleteConfirmation = true">
             <v-list-item-action>
               <v-icon>mdi-trash-can-outline</v-icon>
             </v-list-item-action>
@@ -399,12 +399,18 @@ export default {
     timelineFailed() {
       return this.timelineStatus === 'fail'
     },
+    timelineAvailable() {
+      return this.timelineStatus === 'ready' || (this.showProcessingTimelineEvents && this.timelineStatus === 'processing')
+    },
     timelineChipColor() {
       if (!this.timeline.color.startsWith('#')) {
         return '#' + this.timeline.color
       }
       return this.timeline.color
     },
+    showProcessingTimelineEvents() {
+      return this.$store.state.settings.showProcessingTimelineEvents
+    },
   },
   methods: {
     openDialog() {

timesketch/frontend-ng/src/components/LeftPanel/TimelinesTable.vue

@@ -195,6 +195,14 @@ export default {
         return a.name.localeCompare(b.name)
       })
     },
+    settings() {
+      return this.$store.state.settings
+    },
+  },
+  watch: {
+    'settings.showProcessingTimelineEvents': function () {
+      this.updateEnabledTimelines()
+    },
   },
   methods: {
     isEnabled(timeline) {
@@ -206,6 +214,15 @@ export default {
     disableAllOtherTimelines(timeline) {
       this.$store.dispatch('updateEnabledTimelines', [timeline.id])
     },
+    updateEnabledTimelines() {
+      this.$store.dispatch(
+        'updateEnabledTimelines',
+        this.activeTimelines
+          .filter((tl) => this.$store.state.enabledTimelines.includes(tl.id))
+          .filter((tl) => this.settings.showProcessingTimelineEvents || tl.status[0].status !== 'processing')
+          .map((tl) => tl.id)
+      )
+    },
     timelineStyle(timelineStatus, isSelected) {
       const greyOut = timelineStatus === 'ready' && !isSelected
       return {
@@ -237,10 +254,7 @@ export default {
     }
   },
   created() {
-    this.$store.dispatch(
-      'updateEnabledTimelines',
-      this.activeTimelines.map((tl) => tl.id)
-    )
+    this.updateEnabledTimelines()
   },
   mounted() {
     EventBus.$on('updateCountPerTimeline', this.updateCountPerTimeline)

timesketch/frontend-ng/src/components/SettingsDialog.vue

@@ -40,6 +40,17 @@ limitations under the License.
           >
         </v-list-item-content>
       </v-list-item>
+      <v-list-item>
+        <v-list-item-action>
+          <v-switch v-model="settings.showProcessingTimelineEvents" color="primary" @change="saveSettings()"></v-switch>
+        </v-list-item-action>
+        <v-list-item-content>
+          <v-list-item-title>Show processing timeline events</v-list-item-title>
+          <v-list-item-subtitle
+          >Select to include events from timelines in <strong>processing</strong> state</v-list-item-subtitle
+          >
+        </v-list-item-content>
+      </v-list-item>
     </v-list>
   </v-card>
 </template>
@@ -49,6 +60,7 @@ import ApiClient from '../utils/RestApiClient'
 
 const DEFAULT_SETTINGS = {
   showLeftPanel: true,
+  showProcessingTimelineEvents: false,
 }
 
 export default {
@@ -57,6 +69,7 @@ export default {
       settings: {
         showLeftPanel: true,
         generateQuery: true,
+        showProcessingTimelineEvents: false,
       },
     }
   },

timesketch/lib/utils.py

@@ -526,21 +526,27 @@ def read_and_validate_jsonl(
             )
 
 
-def get_validated_indices(indices, sketch):
+def get_validated_indices(indices, sketch, include_processing_timelines=False):
     """Exclude any deleted search index references.
 
     Args:
         indices: List of indices from the user
         sketch: A sketch object (instance of models.sketch.Sketch).
+        include_processing_timelines: True to include processing
+          timelines, Flase otherwise.
 
     Returns:
         Tuple of two items:
           List of indices with those removed that is not in the sketch
           List of timeline IDs that should be part of the output.
     """
+    allowed_statuses = ["ready"]
+    if include_processing_timelines:
+        allowed_statuses.append("processing")
+
     sketch_structure = {}
     for timeline in sketch.timelines:
-        if timeline.get_status.status.lower() != "ready":
+        if timeline.get_status.status.lower() not in allowed_statuses:
             continue
         index_ = timeline.searchindex.index_name
         sketch_structure.setdefault(index_, [])

timesketch/models/sketch.py

@@ -135,7 +135,7 @@ def get_view_urls(self):
 
     @property
     def active_timelines(self):
-        """List timelines that are ready for analysis.
+        """List timelines that are being processed or ready for analysis.
 
         Returns:
             List of instances of timesketch.models.sketch.Timeline
@@ -144,7 +144,7 @@ def active_timelines(self):
         for timeline in self.timelines:
             timeline_status = timeline.get_status.status
             index_status = timeline.searchindex.get_status.status
-            if (timeline_status or index_status) in ("processing", "fail", "archived"):
+            if (timeline_status or index_status) in ("fail", "archived"):
                 continue
             _timelines.append(timeline)
         return _timelines