Merge pull request #106 from scale8/bot-filter

Added bot filter in API

api/src/backends/databases/GoogleCloudBigQuery.ts

@@ -16,6 +16,7 @@ import BaseLogger from '../logging/abstractions/BaseLogger';
 import { getStorageProviderConfig } from '../../utils/IngestEndpointEnvironmentUtils';
 import { GCBigQueryStreamConfig } from '../../Types';
 import { getBigQueryConfig, getServiceAccountJson } from '../../utils/GoogleCloudUtils';
+import { WebTrafficType } from '../../enums/WebTrafficType';
 
 @injectable()
 export default class GoogleCloudBigQuery extends BaseDatabase {
@@ -57,6 +58,9 @@ export default class GoogleCloudBigQuery extends BaseDatabase {
         return bigQuery;
     }
 
+    protected readonly BOT_TEST =
+        '(browser_name IS NULL OR INSTR(LOWER(browser_name), "bot") > 0 OR INSTR(LOWER(browser_name), "headless") > 0 OR INSTR(LOWER(browser_name), "preview") > 0)';
+
     protected readonly MOBILE_TEST =
         '(INSTR(browser_name, "Mobile") > 0 OR device_name = "iPhone" OR device_name = "iPad" OR os_name = "iOS" OR os_name = "Android")';
 
@@ -278,6 +282,21 @@ export default class GoogleCloudBigQuery extends BaseDatabase {
                       params: { page: queryOptions.filter_options.page },
                   }
                 : undefined;
+        const getTrafficType = () => {
+            if (queryOptions.filter_options.traffic_type === WebTrafficType.BOT) {
+                return {
+                    where: this.BOT_TEST,
+                    params: { traffic_type: queryOptions.filter_options.traffic_type },
+                };
+            } else if (queryOptions.filter_options.traffic_type === WebTrafficType.VISITOR) {
+                return {
+                    where: 'NOT ' + this.BOT_TEST,
+                    params: { traffic_type: queryOptions.filter_options.traffic_type },
+                };
+            } else {
+                return undefined;
+            }
+        };
         const getMobile = () => {
             if (
                 typeof queryOptions.filter_options.mobile === 'boolean' &&
@@ -379,6 +398,7 @@ export default class GoogleCloudBigQuery extends BaseDatabase {
             getReferrer(),
             getReferrerTld(),
             getMobile(),
+            getTrafficType(),
             getBrowser(),
             getBrowserVersion(),
             getScreenSize(),

api/src/backends/databases/MongoDb.ts

@@ -17,6 +17,7 @@ import BaseConfig from '../configuration/abstractions/BaseConfig';
 import GQLError from '../../errors/GQLError';
 import userMessages from '../../errors/UserMessages';
 import BaseLogger from '../logging/abstractions/BaseLogger';
+import { WebTrafficType } from '../../enums/WebTrafficType';
 
 @injectable()
 export default class MongoDb extends BaseDatabase {
@@ -34,6 +35,12 @@ export default class MongoDb extends BaseDatabase {
         ['os_name', /android/i],
     ];
 
+    protected readonly BOT_TEST: [string, RegExp][] = [
+        ['browser_name', /bot/i],
+        ['browser_name', /headless/i],
+        ['browser_name', /preview/i],
+    ];
+
     private getAsMobileAggregationRegex() {
         return this.MOBILE_TEST.map(([input, regex]) => {
             return {
@@ -51,6 +58,12 @@ export default class MongoDb extends BaseDatabase {
         }));
     }
 
+    private getAsBotFilter() {
+        return this.BOT_TEST.map(([input, regex]) => ({
+            [input]: regex,
+        }));
+    }
+
     protected async getCollection(entity: App | IngestEndpoint): Promise<Collection> {
         const entityUsageIngestEndpointEnvironmentId =
             this.getEntityUsageIngestEndpointEnvironmentId(entity);
@@ -215,6 +228,19 @@ export default class MongoDb extends BaseDatabase {
                 return undefined;
             }
         };
+        const getTrafficType = () => {
+            if (queryOptions.filter_options.traffic_type === WebTrafficType.BOT) {
+                return {
+                    $or: this.getAsBotFilter(),
+                };
+            } else if (queryOptions.filter_options.traffic_type === WebTrafficType.VISITOR) {
+                return {
+                    $nor: this.getAsBotFilter(),
+                };
+            } else {
+                return undefined;
+            }
+        };
         const getBrowser = () =>
             MongoDb.getFilterObjectFromStringFilterOption(queryOptions, 'browser', 'browser_name');
         const getBrowserVersion = () =>
@@ -265,6 +291,7 @@ export default class MongoDb extends BaseDatabase {
             getReferrer(),
             getReferrerTld(),
             getMobile(),
+            getTrafficType(),
             getBrowser(),
             getBrowserVersion(),
             getScreenSize(),

api/src/backends/databases/abstractions/BaseDatabase.ts

@@ -4,6 +4,7 @@ import IngestEndpoint from '../../../mongo/models/data/IngestEndpoint';
 import { ObjectId } from 'mongodb';
 import GenericError from '../../../errors/GenericError';
 import { LogPriority } from '../../../enums/LogPriority';
+import { WebTrafficType } from '../../../enums/WebTrafficType';
 
 export interface BaseQueryOptions {
     time_slice: string;
@@ -38,6 +39,7 @@ export interface AppQueryOptions extends BaseQueryOptions {
         browser_version?: string;
         screen_size?: string;
         os?: string;
+        traffic_type?: WebTrafficType;
         event?: string;
         event_group?: string;
         custom_release_id?: string;

api/src/enums/WebTrafficType.ts

@@ -0,0 +1,4 @@
+export enum WebTrafficType {
+    VISITOR = 'VISITOR',
+    BOT = 'BOT',
+}

api/src/gql/TypeDefRegister.ts

@@ -39,6 +39,7 @@ import { Mode } from '../enums/Mode';
 import { InputType } from '../../../common/enums/InputType';
 import { TypeIcon } from '../../../common/enums/TypeIcon';
 import { IngestSchemaWizard } from '../enums/IngestSchemaWizard';
+import { WebTrafficType } from '../enums/WebTrafficType';
 
 @injectable()
 export default class TypeDefRegister {
@@ -233,6 +234,12 @@ export default class TypeDefRegister {
                 value: String!
             }
 
+            """
+            A set of supported \`WebTrafficType\` types.
+            """
+            enum WebTrafficType
+            ${TypeDefRegister.enumToGQL(WebTrafficType)}
+
             """
             A set of supported \`IngestSchemaWizard\` types.
             """

api/src/managers/tag/AppManager.ts

@@ -148,6 +148,7 @@ export default class AppManager extends Manager<App> {
             referrer_tld: String
             page: String
             mobile: Boolean
+            traffic_type: WebTrafficType
             browser: String
             browser_version: String
             screen_size: String

ui/src/components/molecules/ChartPageContainer/AppAnalyticsPageContainer.tsx

@@ -5,6 +5,7 @@ import ChartEventSelector from '../../../lazyComponents/ChartEventSelector';
 import { AppAnalyticsContentProps } from '../../../types/props/AppAnalyticsContentProps';
 import ChartPageContainer, { ChartPageContainerProps, extractFilters } from './ChartPageContainer';
 import ChartBaseFilterSelector from '../../../lazyComponents/ChartBaseFilterSelector';
+import { ChartWebTrafficTypeFilterSelector } from '../../organisms/ChartWebTrafficTypeFilterSelector';
 
 export type AppAnalyticsPageContainerProps = AppAnalyticsContentProps & {
     children: ReactNode;
@@ -21,6 +22,7 @@ const AppAnalyticsPageContainer: FC<AppAnalyticsPageContainerProps> = (
             <>
                 <ChartBaseFilterSelector {...appDashboardContentProps} />
                 <ChartEventSelector {...appDashboardContentProps} />
+                <ChartWebTrafficTypeFilterSelector {...appDashboardContentProps} />
             </>
         ),
         rightHeaderBlock: (

ui/src/components/organisms/ChartWebTrafficTypeFilterSelector.tsx

@@ -0,0 +1,56 @@
+import { FC } from 'react';
+import { FormControl, InputLabel, MenuItem, Select, SelectChangeEvent, Theme } from '@mui/material';
+import { SxProps } from '@mui/system';
+import { AppAnalyticsContentProps } from '../../types/props/AppAnalyticsContentProps';
+import { AppErrorContentProps } from '../../types/props/AppErrorContentProps';
+import { WebTrafficType } from '../../gql/generated/globalTypes';
+import { snakeToTitleCase } from '../../utils/TextUtils';
+
+export const ChartWebTrafficTypeFilterSelector: FC<
+    AppAnalyticsContentProps | AppErrorContentProps
+> = (props: AppAnalyticsContentProps | AppErrorContentProps) => {
+    const { appQueryOptions, setFilter } = props;
+    const currentWebTrafficType = appQueryOptions.filter_options.traffic_type ?? ' ';
+
+    const handleWebTrafficTypeFilterChange = (e: SelectChangeEvent) => {
+        if (e.target.value === ' ') {
+            setFilter('traffic_type', undefined);
+        } else {
+            setFilter('traffic_type', e.target.value as string);
+        }
+    };
+
+    const selectRoot: SxProps<Theme> = {
+        '&:focus': {
+            backgroundColor: '#ffffff',
+        },
+        width: '130px',
+    };
+
+    const selectContainer: SxProps<Theme> = {
+        marginRight: 2,
+        marginBottom: 1,
+    };
+
+    return (
+        <>
+            <FormControl variant="outlined" size="small" sx={selectContainer}>
+                <InputLabel id="bot-label">Traffic Type</InputLabel>
+                <Select
+                    labelId="bot-label"
+                    sx={selectRoot}
+                    value={currentWebTrafficType}
+                    onChange={handleWebTrafficTypeFilterChange}
+                    label="Traffic Type"
+                >
+                    <MenuItem value={' '}>All Traffic</MenuItem>
+                    {Object.keys(WebTrafficType).map((key) => (
+                        <MenuItem key={key} value={key}>
+                            {snakeToTitleCase(key)} Traffic
+                        </MenuItem>
+                    ))}
+                </Select>
+            </FormControl>
+        </>
+    );
+};

ui/src/gql/generated/globalTypes.ts

@@ -381,6 +381,14 @@ export enum VarType {
   TIMESTAMP = "TIMESTAMP",
 }
 
+/**
+ * A set of supported `WebTrafficType` types.
+ */
+export enum WebTrafficType {
+  BOT = "BOT",
+  VISITOR = "VISITOR",
+}
+
 /**
  * In order to use AWS Kinesis as your storage engine, you will need to create an
  * AWS account and create a new service account for Scale8. Please see our
@@ -562,6 +570,7 @@ export interface AppQueryFilterOptions {
   referrer_tld?: string | null;
   page?: string | null;
   mobile?: boolean | null;
+  traffic_type?: WebTrafficType | null;
   browser?: string | null;
   browser_version?: string | null;
   screen_size?: string | null;