Skip to content

Commit

Permalink
Add visualizations to stories (#3129)
Browse files Browse the repository at this point in the history 
* Add visualizations to stories
* Update the Analyzer to support apex aggregations

---------

Co-authored-by: Janosch <[email protected]>
  • Loading branch information
@sydp @jkppr
sydp and jkppr authored Jul 25, 2024
1 parent a92d139 commit 3c7910e
Show file tree
Hide file tree
Showing 3 changed files with 193 additions and 54 deletions.
72 changes: 67 additions & 5 deletions timesketch/frontend-ng/src/views/Story.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ limitations under the License.
</v-btn>
</v-toolbar>
<v-divider></v-divider>
<v-card-text>Aggregations are not yet supported</v-card-text>
<v-card-text>Legacy group Aggregations are not supported. Please view this Story in the old UI or update your analyzer.</v-card-text>
</v-card>
<v-card v-if="block.componentName === 'TsAggregationCompact'" outlined class="mb-2">
<v-toolbar dense flat
Expand All @@ -136,7 +136,7 @@ limitations under the License.
</v-btn>
</v-toolbar>
<v-divider></v-divider>
<v-card-text>Aggregations are not yet supported</v-card-text>
<v-card-text>Legacy aggregations are not supported. Please view this Story in the old UI or update your analyzer.</v-card-text>
</v-card>
<v-card v-if="block.componentName === 'TsCytoscapePlugin'" outlined class="mb-2">
<v-toolbar dense flat>
Expand Down Expand Up @@ -178,6 +178,26 @@ limitations under the License.
<component :is="'TsCytoscape'" v-bind="formatComponentProps(block)"></component>
</v-card-text>
</v-card>
<v-card v-if="block.componentName === 'TsSavedVisualization'" outlined class="mb-2">
<v-toolbar dense flat>
<router-link
style="cursor: pointer; text-decoration: none"
:to="{ name: 'VisualizationView', params: { aggregationId: block.componentProps.savedVisualizationId } }"
>
{{ block.componentProps.name }}
</router-link>

<v-spacer></v-spacer>
<v-btn icon @click="deleteBlock(index)">
<v-icon small>mdi-trash-can-outline</v-icon>
</v-btn>
</v-toolbar>
<v-divider></v-divider>
<v-card-text>
<TsSavedVisualization :aggregationId="block.componentProps.savedVisualizationId">
</TsSavedVisualization>
</v-card-text>
</v-card>
</div>
</div>

Expand All @@ -186,7 +206,7 @@ limitations under the License.
<div class="mb-2 mt-2">
<div
:class="{
hidden: !hover && !block.isActive && !block.showGraphMenu && !block.showSavedSearchMenu && hasContent,
hidden: !hover && !block.isActive && !block.showGraphMenu && !block.showSavedSearchMenu && !block.showSavedVisualizationMenu && hasContent,
}"
>
<!-- Text block -->
Expand Down Expand Up @@ -216,7 +236,7 @@ limitations under the License.
</v-menu>
<v-menu offset-y v-model="block.showGraphMenu">
<template v-slot:activator="{ on, attrs }">
<v-btn rounded outlined small :disabled="!graphPlugins.length" v-bind="attrs" v-on="on">
<v-btn class="mr-2" rounded outlined small :disabled="!graphPlugins.length" v-bind="attrs" v-on="on">
<v-icon left small>mdi-plus</v-icon>
Graphs
</v-btn>
Expand All @@ -240,6 +260,26 @@ limitations under the License.
</v-list>
</v-card>
</v-menu>
<v-menu offset-y v-model="block.showSavedVisualizationMenu">
<template v-slot:activator="{ on, attrs }">
<v-btn rounded outlined small :disabled="!savedVisualizations.length" v-bind="attrs" v-on="on">
<v-icon left small>mdi-plus</v-icon>
Visualizations
</v-btn>
</template>
<v-card>
<v-list>
<v-list-item-group color="primary">
<v-subheader>Saved Visualizations</v-subheader>
<v-list-item v-for="savedVisualization in savedVisualizations" :key="savedVisualization.id">
<v-list-item-content @click="addSavedVisualization(savedVisualization, index)">
{{ savedVisualization.name }}
</v-list-item-content>
</v-list-item>
</v-list-item-group>
</v-list>
</v-card>
</v-menu>
</div>
</div>
</v-hover>
Expand All @@ -257,6 +297,7 @@ import _ from 'lodash'

import TsEventList from '../components/Explore/EventList.vue'
import TsCytoscape from '../components/Graph/Cytoscape.vue'
import TsSavedVisualization from '../components/Visualization/SavedVisualization.vue'

const defaultBlock = () => {
return {
Expand All @@ -268,6 +309,7 @@ const defaultBlock = () => {
isActive: false,
showGraphMenu: false,
showSavedSearchMenu: false,
showSavedVisualizationMenu: false,
}
}

Expand All @@ -279,7 +321,7 @@ const componentCompatibility = () => {

export default {
props: ['sketchId', 'storyId'],
components: { TsEventList, TsCytoscape },
components: { TsEventList, TsCytoscape, TsSavedVisualization },
data: function () {
return {
title: '',
Expand Down Expand Up @@ -310,6 +352,14 @@ export default {
savedGraphs() {
return this.$store.state.savedGraphs
},
savedVisualizations() {
if (!this.$store.state.savedVisualizations) {
return []
}
return this.$store.state.savedVisualizations.filter(
(e) => JSON.parse(e.parameters)['aggregator_class'] === 'apex'
)
},
},
methods: {
updateDraft: _.debounce(function (e, block) {
Expand Down Expand Up @@ -407,6 +457,17 @@ export default {
this.blocks.splice(newIndex, 0, newBlock)
this.save()
},
addSavedVisualization(savedVisualization, index) {
let newIndex = index + 1
let newBlock = defaultBlock()
newBlock.componentName = 'TsSavedVisualization'
newBlock.componentProps = {
name: savedVisualization.name,
savedVisualizationId: savedVisualization.id,
}
this.blocks.splice(newIndex, 0, newBlock)
this.save()
},
editTextBlock(block) {
if (block.edit) {
return
Expand Down Expand Up @@ -440,6 +501,7 @@ export default {
block.isActive = false
block.showGraphMenu = false
block.showSavedSearchMenu = false
block.showSavedVisualizationMenu = false
block.edit = false
if (block.draft) {
block.content = block.draft
Expand Down
101 changes: 72 additions & 29 deletions timesketch/lib/analyzers/browser_search.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,46 +284,89 @@ def run(self):
additional_fields=self._FIELDS_TO_INCLUDE,
)

params = {
"field": "search_string",
"limit": 20,
"index": [self.timeline_id],
top_search_name = f"Top 20 browser search queries ({self.timeline_name})"
top_search_params = {
"aggregator_name": "top_terms",
"aggregator_class": "apex",
"aggregator_parameters": {
"fields": [{"field": "search_string", "type": "text"}],
"aggregator_options": {
"metric": "value_count",
"max_items": 20,
"timeline_ids": [self.timeline_id],
},
"chart_type": "table",
"chart_options": {
"chartTitle": top_search_name,
"height": 600,
"width": 800,
},
},
}
agg_obj = self.sketch.add_aggregation(
name="Top 20 browser search queries ({0:s})".format(self.timeline_name),
agg_name="field_bucket",
agg_params=params,
view_id=view.id,
agg_obj = self.sketch.add_apex_aggregation(
name=top_search_name,
params=top_search_params,
chart_type="table",
description="Created by the browser search analyzer",
label="informational",
view_id=view.id,
)

params = {
"field": "search_day",
"index": [self.timeline_id],
"limit": 20,
top_days_name = f"Top 20 days of search queries ({self.timeline_name})"
top_days_params = {
"aggregator_name": "top_terms",
"aggregator_class": "apex",
"aggregator_parameters": {
"fields": [{"field": "search_day", "type": "text"}],
"aggregator_options": {
"metric": "value_count",
"max_items": 20,
"timeline_ids": [self.timeline_id],
},
"chart_type": "table",
"chart_options": {
"chartTitle": top_days_name,
"height": 600,
"width": 800,
},
},
}
agg_days = self.sketch.add_aggregation(
name="Top 20 days of search queries ({0:s})".format(self.timeline_name),
agg_name="field_bucket",
agg_params=params,
chart_type="table",
agg_days = self.sketch.add_apex_aggregation(
name=top_days_name,
params=top_days_params,
chart_type="bar",
description="Created by the browser search analyzer",
label="informational",
view_id=view.id,
)

params = {
"query_string": 'tag:"browser-search"',
"index": [self.timeline_id],
"field": "domain",
top_engines_name = f"Top 20 Search Engines ({self.timeline_name})"
top_engines_params = {
"aggregator_name": "top_terms",
"aggregator_class": "apex",
"aggregator_parameters": {
"fields": [{"field": "domain", "type": "text"}],
"aggregator_options": {
"metric": "value_count",
"max_items": 20,
"query_string": 'tag:"browser-search"',
"timeline_ids": [self.timeline_id],
},
"chart_type": "bar",
"chart_options": {
"chartTitle": top_days_name,
"height": 600,
"width": 800,
},
},
}
agg_engines = self.sketch.add_aggregation(
name="Top Search Engines ({0:s})".format(self.timeline_name),
agg_name="query_bucket",
agg_params=params,
view_id=view.id,
chart_type="hbarchart",
agg_engines = self.sketch.add_apex_aggregation(
name=top_engines_name,
params=top_engines_params,
chart_type="table",
description="Created by the browser search analyzer",
label="informational",
view_id=view.id,
)

story = self.sketch.add_story(
Expand All @@ -345,7 +388,7 @@ def run(self):
story.add_text("The top 20 most commonly discovered searches were:")
story.add_aggregation(agg_obj)
story.add_text("The domains used to search:")
story.add_aggregation(agg_engines, "hbarchart")
story.add_aggregation(agg_engines)
story.add_text("And the most common days of search:")
story.add_aggregation(agg_days)
story.add_text("And an overview of all the discovered search terms:")
Expand Down
74 changes: 54 additions & 20 deletions timesketch/lib/analyzers/interface.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,55 @@ def __init__(self, sketch_id, analyzer=None):
if not self.sql_sketch:
raise RuntimeError("No such sketch")

def add_apex_aggregation(
self, name, params, chart_type, description="", label=None, view_id=None
):
"""Add aggregation to the sketch using apex charts and tables.
Note: since this is updating the database directly, the caller needs
to ensure the chart type and parameters are valid since no checks are
performed in this function.
Args:
name: the name.
params: a dictionary of the parameters for the aggregation.
chart_type: the chart type.
description: the description, visible in the UI.
label: string with a label to attach to the aggregation.
view_id: optional ID of the view to attach the aggregation to.
"""
if not name:
raise ValueError("Aggregator name needs to be defined.")
if not params:
raise ValueError("Aggregator parameters have to be defined.")

if view_id:
view = View.get_by_id(view_id)
else:
view = None

if "aggregator_name" not in params:
raise ValueError("Aggregator name not specified")
aggregator = params["aggregator_name"]

aggregation = Aggregation.get_or_create(
agg_type=aggregator,
chart_type=chart_type,
description=description,
name=name,
parameters=json.dumps(params, ensure_ascii=False),
sketch=self.sql_sketch,
user=None,
view=view,
)

if label:
aggregation.add_label(label)
db_session.add(aggregation)
db_session.commit()

return aggregation

def add_aggregation(
self,
name,
Expand Down Expand Up @@ -789,37 +838,22 @@ def add_text(self, text, skip_if_exists=False):
block["content"] = text
self._commit(block)

def add_aggregation(self, aggregation, agg_type=""):
def add_aggregation(self, aggregation):
"""Add a saved aggregation to the Story.
Args:
aggregation (Aggregation): Saved aggregation to add to the story.
agg_type (str): string indicating the type of aggregation, can be:
"table" or the name of the chart to be used, eg "barcharct",
"hbarchart". Defaults to the value of supported_charts.
"""
today = datetime.datetime.utcnow()
block = self._create_new_block()
parameter_dict = json.loads(aggregation.parameters)
if agg_type:
parameter_dict["supported_charts"] = agg_type
else:
agg_type = parameter_dict.get("supported_charts")
# Neither agg_type nor supported_charts is set.
if not agg_type:
agg_type = "table"
parameter_dict["supported_charts"] = "table"

block["componentName"] = "TsAggregationCompact"
block["componentProps"]["aggregation"] = {
"agg_type": aggregation.agg_type,
"id": aggregation.id,

block["componentName"] = "TsSavedVisualization"
block["componentProps"] = {
"name": aggregation.name,
"chart_type": agg_type,
"savedVisualizationId": aggregation.id,
"description": aggregation.description,
"created_at": today.isoformat(),
"updated_at": today.isoformat(),
"parameters": json.dumps(parameter_dict),
"user": {"username": None},
}
self._commit(block)
Expand Down

0 comments on commit 3c7910e

Please sign in to comment.