v2.3.0

mods/config.py

@@ -85,7 +85,7 @@ def list_dir(dir, pattern='*.tsv'):
 app_data_remote        = os.path.join(REMOTE_BASE_DIR, 'data')
 app_models_remote      = os.path.join(REMOTE_BASE_DIR, 'models', EXPERIMENT_NAMESPACE)
 app_data               = os.path.join(IN_OUT_BASE_DIR, 'data')
-app_data_features      = os.path.join(app_data, 'features')
+app_data_features      = os.path.join(app_data, 'datapools', 'mods')
 app_models             = os.path.join(IN_OUT_BASE_DIR, 'models', EXPERIMENT_NAMESPACE)
 app_checkpoints        = os.path.join(IN_OUT_BASE_DIR, 'checkpoints', EXPERIMENT_NAMESPACE)
 app_cache              = os.path.join(IN_OUT_BASE_DIR, 'cache', EXPERIMENT_NAMESPACE)

mods/models/mods_model.py

@@ -34,19 +34,19 @@
 import keras
 import numpy as np
 import pandas as pd
-from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, TensorBoard
-from keras.layers import Bidirectional
-from keras.layers import Dense
-from keras.layers import Flatten
-from keras.layers import Input
-from keras.layers import RepeatVector
-from keras.layers.convolutional import Conv1D
-from keras.layers.convolutional import MaxPooling1D
-from keras.layers.recurrent import GRU
-from keras.layers.recurrent import LSTM
-from keras.models import Model
-from keras.optimizers import Adam
-from keras.preprocessing.sequence import TimeseriesGenerator
+from tensorflow.keras.callbacks import Callback, EarlyStopping, ModelCheckpoint, TensorBoard
+from tensorflow.keras.layers import Bidirectional
+from tensorflow.keras.layers import Dense
+from tensorflow.keras.layers import Flatten
+from tensorflow.keras.layers import Input
+from tensorflow.keras.layers import RepeatVector
+from tensorflow.keras.layers import Conv1D
+from tensorflow.keras.layers import MaxPooling1D
+from tensorflow.keras.layers import GRU
+from tensorflow.keras.layers import LSTM
+from tensorflow.keras.models import Model
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
 from keras_self_attention import SeqSelfAttention
 from multiprocessing import Process
 from sklearn.preprocessing import MinMaxScaler

mods/utils.py

@@ -437,6 +437,12 @@ def datapool_read(
 ):
     protocols, merge_on_col = parse_data_specs(data_specs_str)
 
+    if isinstance(time_range, str):
+        logging.info('converting time_range ''%s'' to TimeRange' % time_range)
+        time_range = TimeRange.from_str(time_range)
+        logging.info('time_range converted to %s' % str(time_range))
+
+
     # read dataset from cache
     cache_dir = None
     cache_key = None
@@ -541,12 +547,16 @@ def datapool_read(
                     if protocol not in df_protocol.keys():
                         df_protocol[protocol] = df
                     else:
-                        df_protocol[protocol] = df_protocol[protocol].append(df)
+                        df_protocol[protocol] = pd.concat([df_protocol[protocol], df], axis=0)
+                    logging.info('df_protocol[protocol].shape: %s' % str(df_protocol[protocol].shape))
 
     for ds in protocols:
         protocol = ds['protocol']
+        logging.info('protocol: %s' % protocol)
         # rename columns
         rename_rule = {x[0]: x[1] for x in ds['cols'] if len(x) == 2}
+        logging.info('rename_rule: %s' % str(rename_rule))
+        logging.info('df_protocol.keys(): %s' % str(df_protocol.keys()))
         df_protocol[protocol] = df_protocol[protocol].rename(index=str, columns=rename_rule)
         # convert units:
         # from B to kB, MB, GB use _kB, MB, GB
@@ -573,6 +583,9 @@ def datapool_read(
     df_main = df_main[keep_cols]
     dbg_df(df_main, 'debug', 'df_main', print=False, save=cfg.MODS_DEBUG_MODE)
 
+    logging.info('df_main.shape: %s' % str(df_main.shape))
+    logging.info('df_main.columns: %s' % str(df_main.columns))
+
     # save dataset to cache
     if caching:
         assert cache_dir is not None

requirements.txt

@@ -1,24 +1,23 @@
 # external requirements
-Sphinx==2.4.3
-coverage==5.0.3
-awscli==1.18.13
-python-dotenv==0.5.1
-pandas==0.25.3
-h5py==2.10.0
-python-dateutil==2.8.0
-PyYAML==5.1.2
-keras==2.3.1
-keras-self-attention==0.42.0
-keras-tcn==3.0.1
-gevent==1.4.0
-requests==2.23.0
-numpy==1.18.1
-sympy==1.5.1
-statsmodels==0.11.1
-scikit-learn==0.22.1
-matplotlib==3.0.3
-seaborn==0.9.1
-joblib==0.14.1
-webargs<6.0.0,>=5.5.2
-arrow==0.15.5
-docutils==0.15.2
+Sphinx==5.3.0
+coverage==7.2.7
+awscli==1.27.153
+python-dotenv==1.0.0
+pandas==2.0.2
+h5py==3.7.0
+python-dateutil==2.8.2
+PyYAML==5.4.1
+keras-self-attention==0.51.0
+keras-tcn==3.5.0
+gevent==22.10.2
+requests==2.31.0
+numpy==1.23.4
+sympy==1.12
+statsmodels==0.14.0
+scikit-learn==1.2.2
+matplotlib==3.7.1
+seaborn==0.12.2
+joblib==1.2.0
+webargs==5.5.3
+arrow==1.2.3
+docutils==0.16

setup.cfg

@@ -4,7 +4,7 @@ name = mods
 summary = Intelligent module using ML/DL techniques for underlying IDS and monitoring system
 description-file =
     README.md
-version = 2.1.0
+version = 2.3.0
 author = Giang Nguyen, Stefan Dlugolinsky
 author-email = [email protected], [email protected]
 license = Apache-2