From a3e513e5bc7bdac1489792afb08ea54c4ebd13a4 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 17:53:47 +0000
Subject: [PATCH 01/25] cosmetic and functional fixes

---
 scoring/performance_profile.py | 79 ++++++++++++++++++++++++++--------
 scoring/score_submissions.py   | 40 ++++++++++++-----
 2 files changed, 89 insertions(+), 30 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 8ee271804..f209c8610 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -31,9 +31,14 @@
 import re
 
 from absl import logging
+import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+from tabulate import tabulate
+import re
+
+import logging
 
 from algorithmic_efficiency.workloads.workloads import get_base_workload_name
 import algorithmic_efficiency.workloads.workloads as workloads_registry
@@ -63,6 +68,30 @@
 
 MAX_EVAL_METRICS = ['mean_average_precision', 'ssim', 'accuracy', 'bleu']
 
+#MPL params
+mpl.rcParams['figure.figsize'] = (16, 10)  # Width, height in inches
+mpl.rcParams['font.family'] = 'serif'      
+mpl.rcParams['font.serif'] = ['Times New Roman'] + mpl.rcParams['font.serif']  # Add Times New Roman as first choice
+mpl.rcParams['font.size'] = 22
+mpl.rcParams['savefig.dpi'] = 300  # Set resolution for saved figures
+
+# Plot Elements
+mpl.rcParams['lines.linewidth'] = 3     # Adjust line thickness if needed
+mpl.rcParams['lines.markersize'] = 6      # Adjust marker size if needed
+mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd"])  # Example color cycle (consider ColorBrewer or viridis)
+mpl.rcParams['axes.labelsize'] = 22       # Axis label font size
+mpl.rcParams['xtick.labelsize'] = 20       # Tick label font size
+mpl.rcParams['ytick.labelsize'] = 20
+
+# Legends and Gridlines
+mpl.rcParams['legend.fontsize'] = 20       # Legend font size
+mpl.rcParams['legend.loc'] = 'best'       # Let matplotlib decide the best legend location
+mpl.rcParams['axes.grid'] = True          # Enable grid
+mpl.rcParams['grid.alpha'] = 0.4          # Gridline transparency
+
+def print_dataframe(df):
+      tabulated_df = tabulate(df.T, headers='keys', tablefmt='psql')
+      logging.info(tabulated_df)
 
 def generate_eval_cols(metrics):
   splits = ['train', 'validation']
@@ -177,10 +206,10 @@ def get_workloads_time_to_target(submission,
       num_trials = len(group)
       if num_trials != NUM_TRIALS and not self_tuning_ruleset:
         if strict:
-          raise ValueError(f'Expecting {NUM_TRIALS} trials for workload '
+          raise ValueError(f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
                            f'{workload} but found {num_trials} trials.')
         else:
-          logging.warning(f'Expecting {NUM_TRIALS} trials for workload '
+          logging.warning(f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
                           f'{workload} but found {num_trials} trials.')
 
       # Get trial and time index that reaches target
@@ -194,13 +223,14 @@ def get_workloads_time_to_target(submission,
 
     workloads.append({
         'submission': submission_name,
-        'workload': workload,
+        'workload': re.sub(r'_(jax|pytorch)$', '', workload),
         time_col: np.median(time_vals_per_study),
     })
 
   df = pd.DataFrame.from_records(workloads)
   df = df.pivot(index='submission', columns='workload', values=time_col)
-
+  logging.info("HELLOOOOOOOOO")
+  print_dataframe(df)
   return df
 
 
@@ -269,26 +299,30 @@ def compute_performance_profiles(submissions,
                                      strict))
   df = pd.concat(dfs)
 
+  logging.info("TIME TO TARGET")
+  print_dataframe(df)
+
   # Set score to inf if not within 4x of fastest submission
   best_scores = df.min(axis=0)
   df[df.apply(lambda x: x > 4 * best_scores, axis=1)] = np.inf
 
+  logging.info("4X of budget")
+  print_dataframe(df)
+
   # For each held-out workload if variant target was not hit set submission to inf
   framework = None
   for workload in df.keys():
-    # Check if this is a variant
-    framework = workload.split('_')[-1]
-    workload_ = workload.split(f'_{framework}')[0]
-    if workload_ not in BASE_WORKLOADS:
+    if workload not in BASE_WORKLOADS:
       # If variants do not have finite score set base_workload score to inf
-      base_workload = get_base_workload_name(workload_)
+      base_workload = get_base_workload_name(workload)
       df[base_workload] = df.apply(
-          variant_criteria_filter(base_workload + f'_{framework}', workload),
+          variant_criteria_filter(base_workload, workload),
           axis=1)
+    
+  logging.info("HELDOUT_WORKLOAD FILTER")
+  print_dataframe(df)
 
-  base_workloads = [w + f'_{framework}' for w in BASE_WORKLOADS]
-  df = df[base_workloads]
-  print(df)
+  df = df[BASE_WORKLOADS]
 
   if verbosity > 0:
     logging.info('\n`{time_col}` to reach target:')
@@ -316,11 +350,17 @@ def compute_performance_profiles(submissions,
                            1000):
       logging.info(df)
 
+  logging.info('DIVIDE BY FASTEST')
+  print_dataframe(df)
+
   # If no max_tau is supplied, choose the value of tau that would plot all non
   # inf or nan data.
   if max_tau is None:
     max_tau = df.replace(float('inf'), -1).replace(np.nan, -1).values.max()
 
+  logging.info('AFTER MAYBE SETTING MAX TAU')
+  print_dataframe(df)
+
   if scale == 'linear':
     points = np.linspace(min_tau, max_tau, num=num_points)
   elif scale == 'log':
@@ -375,8 +415,8 @@ def plot_performance_profiles(perf_df,
                               df_col,
                               scale='linear',
                               save_dir=None,
-                              figsize=(30, 10),
-                              font_size=18):
+                              figsize=(30, 10)
+                              ):
   """Plot performance profiles.
 
   Args:
@@ -396,12 +436,13 @@ def plot_performance_profiles(perf_df,
   Returns:
     None. If a valid save_dir is provided, save both the plot and perf_df.
   """
-  fig = perf_df.T.plot(figsize=figsize)
+  fig = perf_df.T.plot(figsize=figsize, alpha=0.7)
   df_col_display = f'log10({df_col})' if scale == 'log' else df_col
   fig.set_xlabel(
-      f'Ratio of `{df_col_display}` to best submission', size=font_size)
-  fig.set_ylabel('Proportion of workloads', size=font_size)
-  fig.legend(prop={'size': font_size}, bbox_to_anchor=(1.0, 1.0))
+      f'Ratio of `{df_col_display}` to best submission')
+  fig.set_ylabel('Proportion of workloads')
+  fig.legend(bbox_to_anchor=(1.0, 1.0))
+  plt.tight_layout()
   maybe_save_figure(save_dir, f'performance_profile_by_{df_col_display}')
   maybe_save_df_to_csv(save_dir,
                        perf_df,
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 0b768855e..4fd5b5b8e 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -22,8 +22,10 @@
 import pandas as pd
 import scoring_utils
 from tabulate import tabulate
+import json
+import pickle
 
-from scoring import performance_profile
+import performance_profile
 
 flags.DEFINE_string(
     'submission_directory',
@@ -101,8 +103,13 @@ def get_summary_df(workload, workload_df, include_test_split=False):
   return summary_df
 
 
-def print_submission_summary(df, include_test_split=True):
+def get_submission_summary(df, include_test_split=True):
+  """Summarizes the submission results into metric and time tables
+  organized by workload.
+  """
+
   dfs = []
+  print(df)
   for workload, group in df.groupby('workload'):
     summary_df = get_summary_df(
         workload, group, include_test_split=include_test_split)
@@ -115,15 +122,26 @@ def print_submission_summary(df, include_test_split=True):
 
 def main(_):
   results = {}
-
-  for submission in os.listdir(FLAGS.submission_directory):
-    experiment_path = os.path.join(FLAGS.submission_directory, submission)
-    df = scoring_utils.get_experiment_df(experiment_path)
-    results[submission] = df
-    summary_df = print_submission_summary(df)
-    with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
-              'w') as fout:
-      summary_df.to_csv(fout)
+  os.makedirs(FLAGS.output_dir, exist_ok=True)
+
+#   for team in os.listdir(FLAGS.submission_directory):
+#     for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
+#         print(submission)
+#         experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
+#         df = scoring_utils.get_experiment_df(experiment_path)
+#         results[submission] = df
+#         summary_df = get_submission_summary(df)
+#         with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
+#                 'w') as fout:
+#             summary_df.to_csv(fout)
+  
+#   # Save results
+#   with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'wb') as f:
+#     pickle.dump(results, f)
+ 
+  # Read results
+  with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'rb') as f:
+    results = pickle.load(f)
 
   if not FLAGS.strict:
     logging.warning(

From e125201e7452f6f659eec8c2738759e34cd26a68 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 17:58:08 +0000
Subject: [PATCH 02/25] formatting

---
 scoring/performance_profile.py | 54 +++++++++++++++++++---------------
 scoring/score_submissions.py   | 30 +++++++++----------
 2 files changed, 45 insertions(+), 39 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index f209c8610..949ec9d45 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -70,28 +70,35 @@
 
 #MPL params
 mpl.rcParams['figure.figsize'] = (16, 10)  # Width, height in inches
-mpl.rcParams['font.family'] = 'serif'      
-mpl.rcParams['font.serif'] = ['Times New Roman'] + mpl.rcParams['font.serif']  # Add Times New Roman as first choice
+mpl.rcParams['font.family'] = 'serif'
+mpl.rcParams['font.serif'] = [
+    'Times New Roman'
+] + mpl.rcParams['font.serif']  # Add Times New Roman as first choice
 mpl.rcParams['font.size'] = 22
 mpl.rcParams['savefig.dpi'] = 300  # Set resolution for saved figures
 
 # Plot Elements
-mpl.rcParams['lines.linewidth'] = 3     # Adjust line thickness if needed
-mpl.rcParams['lines.markersize'] = 6      # Adjust marker size if needed
-mpl.rcParams['axes.prop_cycle'] = mpl.cycler(color=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd"])  # Example color cycle (consider ColorBrewer or viridis)
-mpl.rcParams['axes.labelsize'] = 22       # Axis label font size
-mpl.rcParams['xtick.labelsize'] = 20       # Tick label font size
+mpl.rcParams['lines.linewidth'] = 3  # Adjust line thickness if needed
+mpl.rcParams['lines.markersize'] = 6  # Adjust marker size if needed
+mpl.rcParams['axes.prop_cycle'] = mpl.cycler(
+    color=["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728",
+           "#9467bd"])  # Example color cycle (consider ColorBrewer or viridis)
+mpl.rcParams['axes.labelsize'] = 22  # Axis label font size
+mpl.rcParams['xtick.labelsize'] = 20  # Tick label font size
 mpl.rcParams['ytick.labelsize'] = 20
 
 # Legends and Gridlines
-mpl.rcParams['legend.fontsize'] = 20       # Legend font size
-mpl.rcParams['legend.loc'] = 'best'       # Let matplotlib decide the best legend location
-mpl.rcParams['axes.grid'] = True          # Enable grid
-mpl.rcParams['grid.alpha'] = 0.4          # Gridline transparency
+mpl.rcParams['legend.fontsize'] = 20  # Legend font size
+mpl.rcParams[
+    'legend.loc'] = 'best'  # Let matplotlib decide the best legend location
+mpl.rcParams['axes.grid'] = True  # Enable grid
+mpl.rcParams['grid.alpha'] = 0.4  # Gridline transparency
+
 
 def print_dataframe(df):
-      tabulated_df = tabulate(df.T, headers='keys', tablefmt='psql')
-      logging.info(tabulated_df)
+  tabulated_df = tabulate(df.T, headers='keys', tablefmt='psql')
+  logging.info(tabulated_df)
+
 
 def generate_eval_cols(metrics):
   splits = ['train', 'validation']
@@ -206,11 +213,13 @@ def get_workloads_time_to_target(submission,
       num_trials = len(group)
       if num_trials != NUM_TRIALS and not self_tuning_ruleset:
         if strict:
-          raise ValueError(f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
-                           f'{workload} but found {num_trials} trials.')
+          raise ValueError(
+              f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
+              f'{workload} but found {num_trials} trials.')
         else:
-          logging.warning(f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
-                          f'{workload} but found {num_trials} trials.')
+          logging.warning(
+              f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
+              f'{workload} but found {num_trials} trials.')
 
       # Get trial and time index that reaches target
       trial_idx, time_idx = get_best_trial_index(
@@ -316,9 +325,8 @@ def compute_performance_profiles(submissions,
       # If variants do not have finite score set base_workload score to inf
       base_workload = get_base_workload_name(workload)
       df[base_workload] = df.apply(
-          variant_criteria_filter(base_workload, workload),
-          axis=1)
-    
+          variant_criteria_filter(base_workload, workload), axis=1)
+
   logging.info("HELDOUT_WORKLOAD FILTER")
   print_dataframe(df)
 
@@ -415,8 +423,7 @@ def plot_performance_profiles(perf_df,
                               df_col,
                               scale='linear',
                               save_dir=None,
-                              figsize=(30, 10)
-                              ):
+                              figsize=(30, 10)):
   """Plot performance profiles.
 
   Args:
@@ -438,8 +445,7 @@ def plot_performance_profiles(perf_df,
   """
   fig = perf_df.T.plot(figsize=figsize, alpha=0.7)
   df_col_display = f'log10({df_col})' if scale == 'log' else df_col
-  fig.set_xlabel(
-      f'Ratio of `{df_col_display}` to best submission')
+  fig.set_xlabel(f'Ratio of `{df_col_display}` to best submission')
   fig.set_ylabel('Proportion of workloads')
   fig.legend(bbox_to_anchor=(1.0, 1.0))
   plt.tight_layout()
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 4fd5b5b8e..0ca56a4a8 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -124,21 +124,21 @@ def main(_):
   results = {}
   os.makedirs(FLAGS.output_dir, exist_ok=True)
 
-#   for team in os.listdir(FLAGS.submission_directory):
-#     for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
-#         print(submission)
-#         experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
-#         df = scoring_utils.get_experiment_df(experiment_path)
-#         results[submission] = df
-#         summary_df = get_submission_summary(df)
-#         with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
-#                 'w') as fout:
-#             summary_df.to_csv(fout)
-  
-#   # Save results
-#   with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'wb') as f:
-#     pickle.dump(results, f)
- 
+  #   for team in os.listdir(FLAGS.submission_directory):
+  #     for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
+  #         print(submission)
+  #         experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
+  #         df = scoring_utils.get_experiment_df(experiment_path)
+  #         results[submission] = df
+  #         summary_df = get_submission_summary(df)
+  #         with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
+  #                 'w') as fout:
+  #             summary_df.to_csv(fout)
+
+  #   # Save results
+  #   with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'wb') as f:
+  #     pickle.dump(results, f)
+
   # Read results
   with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'rb') as f:
     results = pickle.load(f)

From ea586fe2f2882b55d2d269905da70ed3f83b9d9b Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 18:22:01 +0000
Subject: [PATCH 03/25] sorting

---
 scoring/performance_profile.py | 4 +---
 scoring/score_submissions.py   | 7 +++----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 949ec9d45..ae5984381 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -26,6 +26,7 @@
   the dictionary of submissions.
 """
 import itertools
+import logging
 import operator
 import os
 import re
@@ -36,9 +37,6 @@
 import numpy as np
 import pandas as pd
 from tabulate import tabulate
-import re
-
-import logging
 
 from algorithmic_efficiency.workloads.workloads import get_base_workload_name
 import algorithmic_efficiency.workloads.workloads as workloads_registry
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 0ca56a4a8..0e99a19c6 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -12,20 +12,19 @@
   --compute_performance_profiles
 """
 
+import json
 import operator
 import os
+import pickle
 
 from absl import app
 from absl import flags
 from absl import logging
 import numpy as np
 import pandas as pd
+import performance_profile
 import scoring_utils
 from tabulate import tabulate
-import json
-import pickle
-
-import performance_profile
 
 flags.DEFINE_string(
     'submission_directory',

From 32121151412c6a27063d44b3ab22d5d61f2a4e24 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 18:34:33 +0000
Subject: [PATCH 04/25] add flags for saving results files

---
 scoring/score_submissions.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 0e99a19c6..4c7f91eaa 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -46,6 +46,16 @@
     'self_tuning_ruleset',
     False,
     'Whether to score on self-tuning ruleset or externally tuned ruleset')
+flags.DEFINE_string(
+    'save_results_to_filename',
+    None,
+    'Filename to save the processed results that are fed into the performance profile functions'
+)
+flags.DEFINE_boolean(
+    'load_results_from_filename',
+    None,
+    'Filename to load processed results from that are fed into performance profile functions'
+)
 FLAGS = flags.FLAGS
 
 

From ffc1ee512e1bcfabc1a3965e7e9fd2b063053e15 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 18:42:22 +0000
Subject: [PATCH 05/25] remove debugging statements

---
 scoring/performance_profile.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index ae5984381..31106f057 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -236,8 +236,6 @@ def get_workloads_time_to_target(submission,
 
   df = pd.DataFrame.from_records(workloads)
   df = df.pivot(index='submission', columns='workload', values=time_col)
-  logging.info("HELLOOOOOOOOO")
-  print_dataframe(df)
   return df
 
 
@@ -306,16 +304,10 @@ def compute_performance_profiles(submissions,
                                      strict))
   df = pd.concat(dfs)
 
-  logging.info("TIME TO TARGET")
-  print_dataframe(df)
-
   # Set score to inf if not within 4x of fastest submission
   best_scores = df.min(axis=0)
   df[df.apply(lambda x: x > 4 * best_scores, axis=1)] = np.inf
 
-  logging.info("4X of budget")
-  print_dataframe(df)
-
   # For each held-out workload if variant target was not hit set submission to inf
   framework = None
   for workload in df.keys():
@@ -325,9 +317,6 @@ def compute_performance_profiles(submissions,
       df[base_workload] = df.apply(
           variant_criteria_filter(base_workload, workload), axis=1)
 
-  logging.info("HELDOUT_WORKLOAD FILTER")
-  print_dataframe(df)
-
   df = df[BASE_WORKLOADS]
 
   if verbosity > 0:
@@ -356,17 +345,11 @@ def compute_performance_profiles(submissions,
                            1000):
       logging.info(df)
 
-  logging.info('DIVIDE BY FASTEST')
-  print_dataframe(df)
-
   # If no max_tau is supplied, choose the value of tau that would plot all non
   # inf or nan data.
   if max_tau is None:
     max_tau = df.replace(float('inf'), -1).replace(np.nan, -1).values.max()
 
-  logging.info('AFTER MAYBE SETTING MAX TAU')
-  print_dataframe(df)
-
   if scale == 'linear':
     points = np.linspace(min_tau, max_tau, num=num_points)
   elif scale == 'log':

From 90c465220195e567e2ab4a7f8f192cc02259ca33 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 18:43:42 +0000
Subject: [PATCH 06/25] add flags

---
 scoring/score_submissions.py | 52 ++++++++++++++++++------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 4c7f91eaa..6bf51d72e 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -47,15 +47,13 @@
     False,
     'Whether to score on self-tuning ruleset or externally tuned ruleset')
 flags.DEFINE_string(
-    'save_results_to_filename',
-    None,
-    'Filename to save the processed results that are fed into the performance profile functions'
-)
+  'save_results_to_filename',
+   None,
+  'Filename to save the processed results that are fed into the performance profile functions.')
 flags.DEFINE_boolean(
-    'load_results_from_filename',
-    None,
-    'Filename to load processed results from that are fed into performance profile functions'
-)
+  'load_results_from_filename',
+  None,
+  'Filename to load processed results from that are fed into performance profile functions')
 FLAGS = flags.FLAGS
 
 
@@ -133,24 +131,26 @@ def main(_):
   results = {}
   os.makedirs(FLAGS.output_dir, exist_ok=True)
 
-  #   for team in os.listdir(FLAGS.submission_directory):
-  #     for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
-  #         print(submission)
-  #         experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
-  #         df = scoring_utils.get_experiment_df(experiment_path)
-  #         results[submission] = df
-  #         summary_df = get_submission_summary(df)
-  #         with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
-  #                 'w') as fout:
-  #             summary_df.to_csv(fout)
-
-  #   # Save results
-  #   with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'wb') as f:
-  #     pickle.dump(results, f)
-
-  # Read results
-  with open(os.path.join(FLAGS.output_dir, 'results.pkl'), 'rb') as f:
-    results = pickle.load(f)
+    # Optionally read results to filename 
+  if FLAGS.load_results_from_filename:
+    with open(os.path.join(FLAGS.output_dir, FLAGS.load_results_from_filename), 'rb') as f:
+      results = pickle.load(f)
+  else:
+    for team in os.listdir(FLAGS.submission_directory):
+      for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
+          print(submission)
+          experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
+          df = scoring_utils.get_experiment_df(experiment_path)
+          results[submission] = df
+          summary_df = get_submission_summary(df)
+          with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
+                  'w') as fout:
+              summary_df.to_csv(fout)
+
+    # Optionally save results to filename
+    if FLAGS.save_results_to_filename:
+      with open(os.path.join(FLAGS.output_dir, FLAGS.save_results_to_filename), 'wb') as f:
+        pickle.dump(results, f)
 
   if not FLAGS.strict:
     logging.warning(

From be6560ebf5d3bc0754a614df10f0599215ea9a95 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Tue, 2 Jul 2024 18:46:09 +0000
Subject: [PATCH 07/25] formatting

---
 scoring/score_submissions.py | 46 ++++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 6bf51d72e..97264748f 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -47,13 +47,15 @@
     False,
     'Whether to score on self-tuning ruleset or externally tuned ruleset')
 flags.DEFINE_string(
-  'save_results_to_filename',
-   None,
-  'Filename to save the processed results that are fed into the performance profile functions.')
+    'save_results_to_filename',
+    None,
+    'Filename to save the processed results that are fed into the performance profile functions.'
+)
 flags.DEFINE_boolean(
-  'load_results_from_filename',
-  None,
-  'Filename to load processed results from that are fed into performance profile functions')
+    'load_results_from_filename',
+    None,
+    'Filename to load processed results from that are fed into performance profile functions'
+)
 FLAGS = flags.FLAGS
 
 
@@ -131,25 +133,33 @@ def main(_):
   results = {}
   os.makedirs(FLAGS.output_dir, exist_ok=True)
 
-    # Optionally read results to filename 
+  # Optionally read results to filename
   if FLAGS.load_results_from_filename:
-    with open(os.path.join(FLAGS.output_dir, FLAGS.load_results_from_filename), 'rb') as f:
+    with open(
+        os.path.join(FLAGS.output_dir, FLAGS.load_results_from_filename),
+        'rb') as f:
       results = pickle.load(f)
   else:
     for team in os.listdir(FLAGS.submission_directory):
-      for submission in os.listdir(os.path.join(FLAGS.submission_directory, team)):
-          print(submission)
-          experiment_path = os.path.join(FLAGS.submission_directory, team, submission)
-          df = scoring_utils.get_experiment_df(experiment_path)
-          results[submission] = df
-          summary_df = get_submission_summary(df)
-          with open(os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
-                  'w') as fout:
-              summary_df.to_csv(fout)
+      for submission in os.listdir(
+          os.path.join(FLAGS.submission_directory, team)):
+        print(submission)
+        experiment_path = os.path.join(FLAGS.submission_directory,
+                                       team,
+                                       submission)
+        df = scoring_utils.get_experiment_df(experiment_path)
+        results[submission] = df
+        summary_df = get_submission_summary(df)
+        with open(
+            os.path.join(FLAGS.output_dir, f'{submission}_summary.csv'),
+            'w') as fout:
+          summary_df.to_csv(fout)
 
     # Optionally save results to filename
     if FLAGS.save_results_to_filename:
-      with open(os.path.join(FLAGS.output_dir, FLAGS.save_results_to_filename), 'wb') as f:
+      with open(
+          os.path.join(FLAGS.output_dir, FLAGS.save_results_to_filename),
+          'wb') as f:
         pickle.dump(results, f)
 
   if not FLAGS.strict:

From 483076b992bd0f12d4e9a2c466ff0a730478259e Mon Sep 17 00:00:00 2001
From: priyakasimbeg <priyakasimbeg@users.noreply.github.com>
Date: Thu, 18 Jul 2024 13:06:34 -0700
Subject: [PATCH 08/25] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f778c94a6..0cb8b7aca 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@
 > [!IMPORTANT]
 > Submitters are no longer required to self-report results. 
 > We are currently in the process of evaluating and scoring received submissions.
-> We are aiming to release results by July 15th 2024.
+> Results coming soon!
 > For other key dates please see [Call for Submissions](CALL_FOR_SUBMISSIONS.md).
 
 ## Table of Contents <!-- omit from toc -->

From 7d01efe6e4bb973c111f7e2a9a696a00ed0c14b0 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Mon, 29 Jul 2024 21:07:52 +0000
Subject: [PATCH 09/25] fixes

---
 scoring/performance_profile.py |  3 +++
 scoring/score_submissions.py   | 35 +++++++++++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 31106f057..19157c2a0 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -227,6 +227,9 @@ def get_workloads_time_to_target(submission,
       else:
         time_val = float('inf')
       time_vals_per_study.append(time_val)
+    num_s = len(time_vals_per_study)
+    print(f'TIME VALS PER STUDY: {num_s}')
+    print(time_vals_per_study)
 
     workloads.append({
         'submission': submission_name,
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 97264748f..5f711d6e4 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -32,7 +32,7 @@
     'Path to submission directory containing experiment directories.')
 flags.DEFINE_string('output_dir',
                     'scoring_results',
-                    'Path to save performance profile table and plot.')
+                    'Path to save performance profile artifacts, submission_summaries and results files.')
 flags.DEFINE_boolean('compute_performance_profiles',
                      False,
                      'Whether or not to compute the performance profiles.')
@@ -51,11 +51,16 @@
     None,
     'Filename to save the processed results that are fed into the performance profile functions.'
 )
-flags.DEFINE_boolean(
+flags.DEFINE_string(
     'load_results_from_filename',
     None,
     'Filename to load processed results from that are fed into performance profile functions'
 )
+flags.DEFINE_string(
+    'exclude_submissions',
+    '',
+    'Optional comma seperated list of names of submissions to exclude from scoring.'
+)
 FLAGS = flags.FLAGS
 
 
@@ -128,6 +133,21 @@ def get_submission_summary(df, include_test_split=True):
   logging.info('\n' + tabulate(df, headers='keys', tablefmt='psql'))
   return df
 
+def compute_leaderboard_score(df, normalize=False):
+  """Compute leaderboard score by taking integral of performance profile.
+
+  Args:
+    df: pd.DataFrame returned from `compute_performance_profiles`.
+    normalize: divide by the range of the performance profile's tau.
+
+  Returns:
+    pd.DataFrame with one column of scores indexed by submission.
+  """
+  scores = np.trapz(df, x=df.columns)
+  if normalize:
+    scores /= df.columns.max() - df.columns.min()
+  return pd.DataFrame(scores, columns=['score'], index=df.index)
+
 
 def main(_):
   results = {}
@@ -144,6 +164,8 @@ def main(_):
       for submission in os.listdir(
           os.path.join(FLAGS.submission_directory, team)):
         print(submission)
+        if submission in FLAGS.exclude_submissions.split(','):
+          continue
         experiment_path = os.path.join(FLAGS.submission_directory,
                                        team,
                                        submission)
@@ -185,10 +207,13 @@ def main(_):
       os.mkdir(FLAGS.output_dir)
     performance_profile.plot_performance_profiles(
         performance_profile_df, 'score', save_dir=FLAGS.output_dir)
-    perf_df = tabulate(
+    performance_profile_str = tabulate(
         performance_profile_df.T, headers='keys', tablefmt='psql')
-    logging.info(f'Performance profile:\n {perf_df}')
-
+    logging.info(f'Performance profile:\n {performance_profile_str}')
+    scores = compute_leaderboard_score(performance_profile_df)
+    scores.to_csv(os.path.join(FLAGS.output_dir, 'scores.csv'))
+    scores_str = tabulate(scores, headers='keys', tablefmt='psql')
+    logging.info(f'Scores: \n {scores_str}')
 
 if __name__ == '__main__':
   # flags.mark_flag_as_required('submission_directory')

From 006b07e4b44390ce1f69fb3c126250ce103d484c Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Mon, 29 Jul 2024 21:10:26 +0000
Subject: [PATCH 10/25] remove logging

---
 scoring/performance_profile.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 19157c2a0..31106f057 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -227,9 +227,6 @@ def get_workloads_time_to_target(submission,
       else:
         time_val = float('inf')
       time_vals_per_study.append(time_val)
-    num_s = len(time_vals_per_study)
-    print(f'TIME VALS PER STUDY: {num_s}')
-    print(time_vals_per_study)
 
     workloads.append({
         'submission': submission_name,

From 4238cb27b469c24d2c4c708ca6239de805b40900 Mon Sep 17 00:00:00 2001
From: Priya Kasimbeg <kasimbeg@google.com>
Date: Mon, 29 Jul 2024 21:12:51 +0000
Subject: [PATCH 11/25] formatting

---
 scoring/score_submissions.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 5f711d6e4..bbc23a1fc 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -30,9 +30,11 @@
     'submission_directory',
     None,
     'Path to submission directory containing experiment directories.')
-flags.DEFINE_string('output_dir',
-                    'scoring_results',
-                    'Path to save performance profile artifacts, submission_summaries and results files.')
+flags.DEFINE_string(
+    'output_dir',
+    'scoring_results',
+    'Path to save performance profile artifacts, submission_summaries and results files.'
+)
 flags.DEFINE_boolean('compute_performance_profiles',
                      False,
                      'Whether or not to compute the performance profiles.')
@@ -133,6 +135,7 @@ def get_submission_summary(df, include_test_split=True):
   logging.info('\n' + tabulate(df, headers='keys', tablefmt='psql'))
   return df
 
+
 def compute_leaderboard_score(df, normalize=False):
   """Compute leaderboard score by taking integral of performance profile.
 
@@ -215,6 +218,7 @@ def main(_):
     scores_str = tabulate(scores, headers='keys', tablefmt='psql')
     logging.info(f'Scores: \n {scores_str}')
 
+
 if __name__ == '__main__':
   # flags.mark_flag_as_required('submission_directory')
   app.run(main)

From 65b40a59412295deb1db06cee8a15549b54f2797 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Tue, 30 Jul 2024 13:04:15 +0200
Subject: [PATCH 12/25] Quality of Life improvements

---
 scoring/performance_profile.py | 26 ++++++++++++++------------
 scoring/score_submissions.py   |  7 ++-----
 scoring/scoring_utils.py       |  2 +-
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 31106f057..391a07927 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -26,20 +26,19 @@
   the dictionary of submissions.
 """
 import itertools
-import logging
 import operator
 import os
 import re
 
-from absl import logging
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+from absl import logging
 from tabulate import tabulate
 
-from algorithmic_efficiency.workloads.workloads import get_base_workload_name
 import algorithmic_efficiency.workloads.workloads as workloads_registry
+from algorithmic_efficiency.workloads.workloads import get_base_workload_name
 from scoring import scoring_utils
 
 WORKLOADS = workloads_registry.WORKLOADS
@@ -184,10 +183,10 @@ def get_workloads_time_to_target(submission,
     if strict:
       raise ValueError(
           f'Expecting {NUM_BASE_WORKLOADS + NUM_VARIANT_WORKLOADS} workloads '
-          f'but found {num_workloads} workloads.')
+          f'but found {num_workloads} workloads for {submission_name}.')
     logging.warning(
         f'Expecting {NUM_BASE_WORKLOADS + NUM_VARIANT_WORKLOADS} workloads '
-        f'but found {num_workloads} workloads.')
+        f'but found {num_workloads} workloads for {submission_name}.')
 
   # For each workload get submission time get the submission times to target.
   for workload, group in submission.groupby('workload'):
@@ -198,11 +197,13 @@ def get_workloads_time_to_target(submission,
     num_studies = len(group.groupby('study'))
     if num_studies != NUM_STUDIES:
       if strict:
-        raise ValueError(f'Expecting {NUM_STUDIES} trials for workload '
-                         f'{workload} but found {num_studies} trials.')
+        raise ValueError(f'Expecting {NUM_STUDIES} studies for workload '
+                         f'{workload} but found {num_studies} studies '
+                         f'for {submission_name}.')
       else:
-        logging.warning(f'Expecting {NUM_STUDIES} trials for workload '
-                        f'{workload} but found {num_studies} trials.')
+        logging.warning(f'Expecting {NUM_STUDIES} studies for workload '
+                        f'{workload} but found {num_studies} studies '
+                        f'for {submission_name}.')
 
     # For each study check trials
     for study, group in group.groupby('study'):
@@ -213,11 +214,13 @@ def get_workloads_time_to_target(submission,
         if strict:
           raise ValueError(
               f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
-              f'{workload} but found {num_trials} trials.')
+              f'{workload} but found {num_trials} trials '
+              f'for {submission_name}.')
         else:
           logging.warning(
               f'In Study {study}: Expecting {NUM_TRIALS} trials for workload '
-              f'{workload} but found {num_trials} trials.')
+              f'{workload} but found {num_trials} trials '
+              f'for {submission_name}.')
 
       # Get trial and time index that reaches target
       trial_idx, time_idx = get_best_trial_index(
@@ -309,7 +312,6 @@ def compute_performance_profiles(submissions,
   df[df.apply(lambda x: x > 4 * best_scores, axis=1)] = np.inf
 
   # For each held-out workload if variant target was not hit set submission to inf
-  framework = None
   for workload in df.keys():
     if workload not in BASE_WORKLOADS:
       # If variants do not have finite score set base_workload score to inf
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index bbc23a1fc..bc2340029 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -12,18 +12,15 @@
   --compute_performance_profiles
 """
 
-import json
 import operator
 import os
 import pickle
 
-from absl import app
-from absl import flags
-from absl import logging
 import numpy as np
 import pandas as pd
 import performance_profile
 import scoring_utils
+from absl import app, flags, logging
 from tabulate import tabulate
 
 flags.DEFINE_string(
@@ -136,7 +133,7 @@ def get_submission_summary(df, include_test_split=True):
   return df
 
 
-def compute_leaderboard_score(df, normalize=False):
+def compute_leaderboard_score(df, normalize=True):
   """Compute leaderboard score by taking integral of performance profile.
 
   Args:
diff --git a/scoring/scoring_utils.py b/scoring/scoring_utils.py
index 0dd997ab9..fd8b0b2c3 100644
--- a/scoring/scoring_utils.py
+++ b/scoring/scoring_utils.py
@@ -4,8 +4,8 @@
 import os
 import re
 
-from absl import logging
 import pandas as pd
+from absl import logging
 
 import algorithmic_efficiency.workloads.workloads as workloads_registry
 

From 9b6c845d535b012294997710b080b33a7a6fdb96 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Tue, 30 Jul 2024 15:55:59 +0200
Subject: [PATCH 13/25] Fix import order

---
 scoring/performance_profile.py | 4 ++--
 scoring/score_submissions.py   | 4 +++-
 scoring/scoring_utils.py       | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 391a07927..372684fe2 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -30,15 +30,15 @@
 import os
 import re
 
+from absl import logging
 import matplotlib as mpl
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-from absl import logging
 from tabulate import tabulate
 
-import algorithmic_efficiency.workloads.workloads as workloads_registry
 from algorithmic_efficiency.workloads.workloads import get_base_workload_name
+import algorithmic_efficiency.workloads.workloads as workloads_registry
 from scoring import scoring_utils
 
 WORKLOADS = workloads_registry.WORKLOADS
diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index bc2340029..22b978fd7 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -16,11 +16,13 @@
 import os
 import pickle
 
+from absl import app
+from absl import flags
+from absl import logging
 import numpy as np
 import pandas as pd
 import performance_profile
 import scoring_utils
-from absl import app, flags, logging
 from tabulate import tabulate
 
 flags.DEFINE_string(
diff --git a/scoring/scoring_utils.py b/scoring/scoring_utils.py
index fd8b0b2c3..0dd997ab9 100644
--- a/scoring/scoring_utils.py
+++ b/scoring/scoring_utils.py
@@ -4,8 +4,8 @@
 import os
 import re
 
-import pandas as pd
 from absl import logging
+import pandas as pd
 
 import algorithmic_efficiency.workloads.workloads as workloads_registry
 

From 1e6b0659c17a24404fd4dd69d005bda6c40d1968 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Wed, 31 Jul 2024 16:26:01 +0200
Subject: [PATCH 14/25] Adds functionality to compute speedups

Geometric means across individual workload speedups between two algorithms.
---
 scoring/compute_speedups.py | 112 ++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 scoring/compute_speedups.py

diff --git a/scoring/compute_speedups.py b/scoring/compute_speedups.py
new file mode 100644
index 000000000..5fb5f259d
--- /dev/null
+++ b/scoring/compute_speedups.py
@@ -0,0 +1,112 @@
+"""File to compute speedups (i.e. geometric means between runtimes)."""
+
+import pickle
+
+from absl import app
+from absl import flags
+import numpy as np
+import pandas as pd
+from performance_profile import BASE_WORKLOADS
+from performance_profile import get_workloads_time_to_target
+from scipy import stats
+
+flags.DEFINE_string('results_txt', None, 'Path to full scoring results file.')
+flags.DEFINE_string(
+    'base',
+    'prize_qualification_baseline',
+    'Base submission to compare to. Defaults to the `prize_qualification_baseline`.'
+)
+flags.DEFINE_string('comparison', None, 'Submission to compute the speedup of.')
+flags.DEFINE_boolean('self_tuning_ruleset',
+                     False,
+                     'Whether the self-tuning ruleset is being scored.')
+flags.DEFINE_boolean('save_results',
+                     False,
+                     'Whether to save the results to disk.')
+FLAGS = flags.FLAGS
+
+MAX_BUDGETS = {
+    'criteo1tb': 7703,
+    'fastmri': 8859,
+    'imagenet_resnet': 63_008,
+    'imagenet_vit': 77_520,
+    'librispeech_conformer': 61_068,
+    'librispeech_deepspeech': 55_506,
+    'ogbg': 18_477,
+    'wmt': 48_151,
+}
+
+
+def replace_inf(row):
+  """Replace ifs with maximum runtime budget (+1 second).
+
+  Args:
+      row (pd.Series): The original row.
+
+  Returns:
+      pd.Series: The row with infs replaced.
+  """
+  workload_name = row.name
+  # Factor of 3 for self-tuning ruleset
+  factor = 3 if FLAGS.self_tuning_ruleset else 1
+  max_runtime_workload = factor * MAX_BUDGETS[workload_name]
+  row.replace(np.inf, max_runtime_workload + 1, inplace=True)
+  return row
+
+
+def compute_speedup():
+  """Compute speedup between two algorithms."""
+  # Load results from disk
+  with open(FLAGS.results_txt, 'rb') as f:
+    results = pickle.load(f)
+
+  # Compute median over runtimes for both training algorithms
+  base_results = get_workloads_time_to_target(
+      results[FLAGS.base],
+      FLAGS.base,
+      time_col="score",
+      self_tuning_ruleset=FLAGS.self_tuning_ruleset,
+  )
+  comparison_results = get_workloads_time_to_target(
+      results[FLAGS.comparison],
+      FLAGS.comparison,
+      time_col="score",
+      self_tuning_ruleset=FLAGS.self_tuning_ruleset,
+  )
+
+  # Merge results
+  merged_results = pd.concat([base_results, comparison_results]).transpose()
+
+  # Ignore workload variants (only consider base workloads) for speedup
+  merged_results = merged_results.loc[merged_results.index.isin(BASE_WORKLOADS)]
+
+  # Replace infs with maximum runtime budget (+1 second)
+  merged_results = merged_results.apply(replace_inf, axis=1)
+
+  # Compute speedup
+  merged_results['speedup'] = merged_results[
+      f'{FLAGS.comparison}'] / merged_results[f'{FLAGS.base}']
+  speedups = merged_results['speedup'].to_numpy()
+  mean_speedup = stats.gmean(speedups)  # Geometric mean over workload speedups
+
+  print(merged_results, end='\n\n')
+  print(
+      f"Average speedup of {FLAGS.comparison} compared to {FLAGS.base}: {mean_speedup} or roughly {(1-mean_speedup):.1%}"
+  )
+
+  if FLAGS.save_results:
+    # Optionally save results to disk
+    print("Saving results to disk...")
+    filename = f'{FLAGS.comparison}_vs_{FLAGS.base}_speedup_{(1-mean_speedup):.1%}.csv'
+    merged_results.to_csv(filename)
+
+
+def main(_):
+  """Main function to compute speedup between two algorithms."""
+  compute_speedup()
+
+
+if __name__ == '__main__':
+  flags.mark_flag_as_required('results_txt')
+  flags.mark_flag_as_required('comparison')
+  app.run(main)

From 3da063df83df5018ff64e4be9639ea7c8662679e Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Wed, 31 Jul 2024 16:26:38 +0200
Subject: [PATCH 15/25] Fix: Only consider workload variant times if they
 trained the base workload

---
 scoring/performance_profile.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index 372684fe2..b77d69d05 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -307,6 +307,14 @@ def compute_performance_profiles(submissions,
                                      strict))
   df = pd.concat(dfs)
 
+  # For each held-out workload set to inf if the base workload is inf
+  for workload in df.keys():
+    if workload not in BASE_WORKLOADS:
+      # If base do not have finite score set variant score to inf
+      base_workload = get_base_workload_name(workload)
+      df[workload] = df.apply(
+          variant_criteria_filter(workload, base_workload), axis=1)
+
   # Set score to inf if not within 4x of fastest submission
   best_scores = df.min(axis=0)
   df[df.apply(lambda x: x > 4 * best_scores, axis=1)] = np.inf

From 5168eb5e2403e30321cf7c685c243b3147c11a16 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Wed, 31 Jul 2024 16:45:04 +0200
Subject: [PATCH 16/25] Fix max_tau to 4.0

---
 scoring/score_submissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 22b978fd7..02ad82fc0 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -198,7 +198,7 @@ def main(_):
         results,
         time_col='score',
         min_tau=1.0,
-        max_tau=None,
+        max_tau=4.0,
         reference_submission_tag=None,
         num_points=100,
         scale='linear',

From d6d6239f436a776d0f5cd908412837a93c1cc566 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Thu, 29 Aug 2024 13:22:19 +0200
Subject: [PATCH 17/25] Update gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 95d9fa6c1..d2e212366 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ algorithmic_efficiency/workloads/librispeech_conformer/work_dir
 *.vocab
 wandb/
 *.txt
+scoring/plots/
 
 !scoring/test_data/experiment_dir/study_0/mnist_jax/trial_0/eval_measurements.csv
 !scoring/test_data/experiment_dir/study_0/mnist_jax/trial_1/eval_measurements.csv
\ No newline at end of file

From 414e82e21fc53960f53a0dec02273b9a5e167cd5 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Thu, 29 Aug 2024 13:31:20 +0200
Subject: [PATCH 18/25] Fix scoring bug handeling nan values

---
 scoring/performance_profile.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py
index b77d69d05..32acae9ab 100644
--- a/scoring/performance_profile.py
+++ b/scoring/performance_profile.py
@@ -26,6 +26,7 @@
   the dictionary of submissions.
 """
 import itertools
+import json
 import operator
 import os
 import re
@@ -45,6 +46,10 @@
 BASE_WORKLOADS = workloads_registry.BASE_WORKLOADS
 WORKLOAD_NAME_PATTERN = '(.*)(_jax|_pytorch)'
 BASE_WORKLOADS_DIR = 'algorithmic_efficiency/workloads/'
+# Open json file to read heldout workloads
+# TODO: This probably shouldn't be hardcoded but passed as an argument.
+with open("held_out_workloads_algoperf_v05.json", "r") as f:
+  HELDOUT_WORKLOADS = json.load(f)
 # These global variables have to be set according to the current set of
 # workloads and rules for the scoring to be correct.
 # We do not use the workload registry since it contains test and development
@@ -248,6 +253,9 @@ def filter(x):
     try:
       if x[variant_workload] == np.inf:
         return np.inf
+      # Also check for nan values (e.g. OOMs)
+      elif np.isnan(x[variant_workload]):
+        return np.inf
       else:
         return x[base_workload]
     except KeyError as e:
@@ -306,8 +314,14 @@ def compute_performance_profiles(submissions,
                                      self_tuning_ruleset,
                                      strict))
   df = pd.concat(dfs)
-
-  # For each held-out workload set to inf if the base workload is inf
+  # Restrict to base and sampled held-out workloads
+  # (ignore the additional workload variants of the baseline
+  # as they cause issues when checking for nans in workload variants).
+  df = df[BASE_WORKLOADS + HELDOUT_WORKLOADS]
+  # Sort workloads alphabetically (for better display)
+  df = df.reindex(sorted(df.columns), axis=1)
+
+  # For each held-out workload set to inf if the base workload is inf or nan
   for workload in df.keys():
     if workload not in BASE_WORKLOADS:
       # If base do not have finite score set variant score to inf
@@ -319,14 +333,13 @@ def compute_performance_profiles(submissions,
   best_scores = df.min(axis=0)
   df[df.apply(lambda x: x > 4 * best_scores, axis=1)] = np.inf
 
-  # For each held-out workload if variant target was not hit set submission to inf
+  # For each base workload if variant target was not hit set submission to inf
   for workload in df.keys():
     if workload not in BASE_WORKLOADS:
       # If variants do not have finite score set base_workload score to inf
       base_workload = get_base_workload_name(workload)
       df[base_workload] = df.apply(
           variant_criteria_filter(base_workload, workload), axis=1)
-
   df = df[BASE_WORKLOADS]
 
   if verbosity > 0:

From 5069c5fe901bf79864f59cc0f31de494f0cb8e27 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Wed, 4 Sep 2024 11:54:21 +0200
Subject: [PATCH 19/25] Update news note

---
 README.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0cb8b7aca..5a1f10a33 100644
--- a/README.md
+++ b/README.md
@@ -27,10 +27,7 @@
 ---
 
 > [!IMPORTANT]
-> Submitters are no longer required to self-report results. 
-> We are currently in the process of evaluating and scoring received submissions.
-> Results coming soon!
-> For other key dates please see [Call for Submissions](CALL_FOR_SUBMISSIONS.md).
+> The results of the inaugural AlgoPerf: Training Algorithms benchmark competition have been announced. See the [MLCommons blog post](https://mlcommons.org/2024/08/mlc-algoperf-benchmark-competition/) for an overview and the [results page](https://mlcommons.org/benchmarks/algorithms/) for more details on the results. We are currently preparing an in-depth analysis of the results in the form of a paper and plan the next iteration of the benchmark competition.
 
 ## Table of Contents <!-- omit from toc -->
 

From 1a3de5bf5fa2864870e53ec3d17b2553de4a09e4 Mon Sep 17 00:00:00 2001
From: Frank Schneider <frank.stefan.schneider@gmail.com>
Date: Wed, 4 Sep 2024 11:55:25 +0200
Subject: [PATCH 20/25] Update results announcement time

---
 CALL_FOR_SUBMISSIONS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CALL_FOR_SUBMISSIONS.md b/CALL_FOR_SUBMISSIONS.md
index 0e21f0e9c..c5fff46d7 100644
--- a/CALL_FOR_SUBMISSIONS.md
+++ b/CALL_FOR_SUBMISSIONS.md
@@ -17,7 +17,7 @@ Submissions can compete under two hyperparameter tuning rulesets (with separate
 - **Registration deadline to express non-binding intent to submit: February 28th, 2024**.\
 Please fill out the (mandatory but non-binding) [**registration form**](https://forms.gle/K7ty8MaYdi2AxJ4N8).
 - **Submission deadline: April 04th, 2024** *(moved by a week from the initial March 28th, 2024)*
-- [tentative] Announcement of all results: July 15th, 2024
+- [Announcement of all results](https://mlcommons.org/2024/08/mlc-algoperf-benchmark-competition/): August 1st, 2024
 
 For a detailed and up-to-date timeline see the [Competition Rules](/COMPETITION_RULES.md).
 

From d09fab5d8e682a278f4654892330ccc4f890b354 Mon Sep 17 00:00:00 2001
From: Niccolo-Ajroldi <n.ajroldi@gmail.com>
Date: Fri, 11 Oct 2024 17:37:11 +0200
Subject: [PATCH 21/25] fixed :bug in summary_df

---
 scoring/score_submissions.py                    |  14 ++++++++++++--
 .../performance_profile_by_score.pdf            | Bin 0 -> 18888 bytes
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 scoring/scoring_results/performance_profile_by_score.pdf

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 02ad82fc0..4ce1dee16 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -88,8 +88,18 @@ def get_summary_df(workload, workload_df, include_test_split=False):
   summary_df['time to best eval on val (s)'] = workload_df.apply(
       lambda x: x['accumulated_submission_time'][x['index best eval on val']],
       axis=1)
-  summary_df['time to target on val (s)'] = summary_df.apply(
-      lambda x: x['time to best eval on val (s)']
+  workload_df['val target reached'] = workload_df[validation_metric].apply(
+      lambda x: target_op(x, validation_target)).apply(np.any)
+  workload_df['index to target on val'] = workload_df.apply(
+      lambda x: np.argmax(target_op(x[validation_metric], validation_target))
+      if x['val target reached'] else np.nan,
+      axis=1)
+  summary_df['time to target on val (s)'] = workload_df.apply(
+      lambda x: x['accumulated_submission_time'][int(x['index to target on val'])]
+      if x['val target reached'] else np.inf,
+      axis=1)
+  summary_df['step to target on val'] = workload_df.apply(
+      lambda x: x['global_step'][int(x['index to target on val'])]
       if x['val target reached'] else np.inf,
       axis=1)
 
diff --git a/scoring/scoring_results/performance_profile_by_score.pdf b/scoring/scoring_results/performance_profile_by_score.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..4d654f2ba0920a39821072a085332dd8344835a6
GIT binary patch
literal 18888
zcmch92RxNu_;^~lRzya|jVOhCuj{%-L{|2WWbZw%y&_RYWQ(lq6_TAT*(*_^sH`#~
zq2+&G^!=9ne&0U-&;RrH{hr?QKJRnRdCqg5b)NS<msLqzk`u+n17j_C3D2*Fq2Nfk
zouMV{+&MTx&E4Jvju11z8Ccty!4XOZW+qN>Zh)Wy7Z!$@*cwBKe7`i1w6n#*(OU=z
zO+`aX6C)fPyL~I+hLcjo8Q@IdJlg;z102r8(H4$@Zea*ja|2@wTQfNC&QnJ_BUKX|
z{0bOWToRCC;)a7Gq-_8R7j~{<J6CDA&M)Mk{+s|rH$=Uo37~G9e1wXLlby4p5tt7|
zzX}{7Z(?j=aKX+EjEDsPaie&Va4a_xt^-4efmwm3PH_HhR8d=7J8*}B@P6e4^!O+1
z<V<YMaOQB-&Tx_z)?l)5grqg#fVhc~ov{f-vlGtI#J~pTmNb9&iqq3e$F9HnsxgCd
zzrnZGX%tVS?tbT_T6gnPN(VWyBovJ`LCm)=m*>hKuDu95j-?U1VH!;Pt+4#5+1l+-
zxfKBm-@U(&wtxH*gufP4|Gl#M)OiNIf^SP_f1G1@3I466{kF8vH@S%3$D<rpC+EvQ
z_9HWJ_P)mEW~b$<vGd}U+V%Yi_hL(LoI{RD7he6W6^YSG-1(HA^KrV5BleXeQhZ->
z%)d*eo|FwSvJv~}U0}ecN4CkBOtw}>GDz3F#4vtpd}U;*XiT$u<x%Aeit%7t!*(Ms
z{&yUU3Vq*5FP$%{W70L;a1cz-_SndQJrO*n{$?htq-fz@hTSE^$9miK#|Heut0mEM
zUe1F=PIN4i8Y}kSterO}=El!|zj{a5tCam~b6fr<vED|%x6n6NxAn0(CNhmzcH=MF
zPAkA}T;V9`Vo{k%o&2EpbhTBQ%i!emwP#t>;{|*hT*0cAjEO7V74G6N2DPN`A=Pz!
ztZOATG>K;iiR@I8?6K%<DePxt_|$USwczE?654v93uxvP*~;g<lnbVJ++M+AgL|bO
zr$0BSiFf6&?o~MK8Q0&Ys~S9b)=nkO4eQjJ{Z`7$w&Wyjv|*6?3<J?<#c@)1;;PR|
z-#&FDs?XR?^<^EzOo*hDayF5klbhEYNruHtJ=P3yeW-Z9R{A<c?}>+Hbb}*Ku7jWH
zzTD`2!oj{2>9g*!&UWzW$K`s5^|_g$_UHDWjn`)<*FSiDA6%NVUU&EYXj$p@edR$^
zNJ!L!Y1fal)^W2Asb>mn7GhqXc6&fJ&Dm+Gbi+KhY^C2A8%QH0^Zfmc(v!Y{PcPn|
zr*Xv|Uvo)tvokt_cVDShc`0JDue7{%f_0u<p@D5YxKlX$$Y+gHFLj&^3Ei$ek15Ez
z_xi+8nu$$*v+N7iyEjOx#cR1U;@31zjp;b61kG1Z&2))2isgxQzPJ<=SC%b6cul4#
z<C&`E*L>B5B7wwAo<&2AAu!Aw7-k3zldJI2;M2owEtThbNsrsES)g9Y5w)PIUw=Af
zd_KMIcv#I3{iXXW52($iN4}p=`Mhb@)3fR7;qqyH(t9o$H})oL&S&7sr3>9><yIXx
zQi3z8U()er^#w685`XiXc_1WA$0Jvw<c+Fv@G7OTv>Kv6?Z3vyEs%3#BJEatP+#5z
zVTwoCqwK~vMk|<u*m*)RTNd8w;AVa&`pL6$eaAyBufLp#>Ux8N1=a_09}GRhq(VGc
zbaL_=$(aNr?sJnt(Mv_dZKvMrEIfbhAD;a*GI4@r`b-)NFTE6>by-o?iLCwEHk#CY
zXRhU4hgF#{(&fdfVEH5{NuttdZ#m`BoULtH3O@5mw^qldjd!0BvFXG0Z%PmO<|7I5
zDbq6Px-FeJPSbUmUP?rlpADLCEI39bHYC87I}!hu>x*^}WwE&2+jHK5Rrd&l(gnIc
zK2|OZct;~ln}9f^8tg8UL-&%IMpr+u-QBJ@sg=7;0ZVBuY5H)8ijp(P{8C8njl7Am
z_@QT-$`1osuDDP+%V<snIMEBaXt|NB3_K3a4P|ew6;=J1W+a1J%-P?8zKgIJ|CUiS
z<a*5^7#3nPR#48WhBrOaQ&3J7o_>9%+Y-g8jCndzRQ^Upm%A&@zKv{Yg0kDvQ-n%u
zt-YYUEwHftq4lK@zjRAHPu*r&sUejdG23`t;UuL+(}+}R3xjHKr7Nk)OpbF2UQeIF
zH}a^CgXCYmKCB}1<*w$2edi>#2o$q0lzF~bP;)7|D4_Vk0;6)xi+eg)UM{u!amK^*
z2Riz_yJFjJElb4BSEEiTpQp#0ug+Fo3_V~pcDW*{%IOF_Y{2t<>{o6>>gXCB*Bf|L
zq5aRDg97}^KGcG4&PNwN1--sdYKl#NCp1CXzhOSiz(ATqE{LhI{cvBoG5&!L_A}++
zk8_kY3-Kjqo7>GF)_SNF$K0(t_VH+WbR~MYcs2CM<qGt0<o(kI)`Cu)@~6m8m<bN+
zd*<kq2s)8?TB#j5{;sFmD(HGuDZoaE5|+3=c;CZl>6bTB6-bfhU$xNnu(l-gj?0u*
z&xj++#&wGBv*&vI!@E_7(?kg!USn%3Z@{vO(Zj)ZIr<Ku%L9^!tX5s-q67RJ60a57
zH2N`UU9B>?c3AmSnuOrdr_@xG+FA>qgiIffUb%AZ?PR4&nir{5OPFhgt7ft}#bw5g
z;L>vfDrY;5xMJJ1Il`+n8CRY!(}cVu*Z6tQ5ZMX=P}C5Pl=kHvgzbB%%_Gb5bB_((
zyIPmyx6yu3Kl}~A0{Yh6>f5R0+!Vkx>treT{oaTJbbTe6uw_{e9&6_Re3_WK6YZx7
z0r-@50RToDcr;<?&UWM{k{t311fV~?3ji?Qh+N+4cN^p1CVZf=Hb?Klf4ed5G%ETH
zuuaYWl5;TOmgScjmT`voZD$9U843`}%LzV(Qrh02)IDP<sCzc3O~~X#E<DBT)nzte
z#L?8I9S~kcz~sRC`#nIJWcCV`SdR}`LQzi`aW%Dh9TR@c?9~$U`+b+h`i--{JO}sN
zO;|t^&8@!w8IGmk_j_n~PYkOU+mQ*hA3+MC=>%Fkp!Xm%1b8Dt_&We#FyS+M1%@mo
zi2OuCH*!D#htypFfDwLRyWee$f1A7kjqS;WkKXvFoA~SDRTRGhcBm0wA9v~Q@`UM)
zdff{Wx)iXKHYa3ouT|OPi^1Tl)}pEJbrbgv=N=Qb%sDf`QWOxrUh}o`Q+GN6lM%VF
zZBpGP_9sWZHUjO|=>c|fGKKKegQfn8_c9QZ$ThaP|H*+cUzp2bsExJgOJ$2XlZF$h
z7Hn1HD(Bu%Xn1Q+4zy(k=Izw;!ezBr9d07LUuK5ad~%2rUWj|q=B#Q^-Z7j_wDvOQ
zhdlLy%~Qlg1FrMclDF}M;8N40_W_H2L~FLv>+uNcg|XZv@AOzMJ<)7HwccveBZ>86
z6ITN4$m=B5Z6(%)zbTNIh+wM&!|N@NZY?d4k&Qy%W>?v!3hm~#Q(pq@s&@as*jXF6
z_-OIz`bex4i`te>)-fLG+|rlJjQ{8pxZhD<2NOuYpy$*09XlGihXVfWV23+kvO1p~
zhz?KYy)SyOqw7$I8`kcjt;;v8{!}ZVos3|B9mV^pR!`4&O8kG-&DHx6v#%dKQhjq&
zr*dt0?hTjTdrs;F@BSt!x7jy&`m=BD{8Gjr-=*_iAX+Q>O1&1Jf5qu0R5}Z*0)`5!
zT6W5v_kY5mXLkTD;m<ZduRCq;oSk5?*p|8tjvejZk-~rOx3yggEO`k5)7{|H{|Eqw
zSHM5I4++Fp2RE#?RX0|6VW7=q;>wpw$#qbddjG~JQV<jC8^Y@Avw5tujY>H35HK;o
zIMlDCXqs{~N$p{WDuH?cyA(&X7E84Dk3_}_VuZinrDNJipo5H0%JuM%)J|3laqM2!
zx}AVeG86a8&p*WRD{<<OW@L71?^M-f<MxjYyc5x??oyYHJ1_*Q+fQKNNtNZSW2#m)
zeNHSFeqLs$Ul}eTKbGrIHK^`9x%dwf<;-tLTlm{Zl%P2<ZbyIi0-$hCoylK%`Q-^1
zPT=*aHhF<liea9k4`zyz*({UC!&;XP)Thh~FCD-CeDlWA#-X_S<<BNBzw+8KIbsv0
zKc&{l+ifi8zFQ~xa6C)u?BqJrg<y<bvf-`n+L~GX)>8%nj*TnG3g>;=Q*m#ayB=0Q
zF4KPW*)7w}&%<V%4Mo55Qu5mBTyb2AZr=EFR{rE5dqRa%S9Q*mKcKk}?<+@^;zSK}
zdp@&AE!!$mCE;gL-_Ki$m0?s=w5YH52>MEfUs)f7v>oShS>>`#M%R4wA^Q0Elx(p*
znqhJXXRk3hv3xV=OG!DWjg{8qke%M<w<hZ8Q@#aEkG#Bb9?Ngvx_RuIUy(Y`q*I`%
zn0$YA5MQ0a#ZH^<^Q%f8GWr@EJ>F^#;wIg)94V7sXrkUDpJM8^`U0kUVV(0Hj&-`{
z1w6byedta`4YXF_YnKa>Q5&_Sb>q{Oh+xb~$e6G$O>y<!gK71A=RLiaKdn8w7ALD2
zc!0~GitZdK%Lx9;$o2L<jG)`wzL%YPbv~~ZhkJ*o<HrDbcETSY8O3t-*o#n`F*+qH
z*n4D<ToJ`sf=*q6HR`>;SEeXl^DgoA@a?nZc=NQ*RcL@y>0a&rpm2MEcON_@UA=H0
zC%wL$)2kKOn1AM~mS{E;`}l!|C@pj2wV;G7r;d5M#pjN`jni%}pJ(RD@ao@rUuv3V
z%ZR}jMq_MSNtr(5C=xoxm6<kuygVGg&3yXCvT%<=f!)mRj?`Ht5{>5OwvY!cXY(sv
z8nUJAZZQ_7FnG6oS8bv-!hL(LdjfMw6hE9Ydw%qlcN?Exoo@Yj#gNZTorM5(=pcU`
zW4=GPmVZ#VtH7htAD>rQggsmylR4^sXF^h@y@E0!J3INEoXFO;A4&!AAknCwiIS~+
z0R+4Pk_tZ&q3p=+oB<yOBnti`VIUDMNYRWW5cj<6&qDAqs`rf?+jXx^ULu(f7bZzK
ziromDxtpgcDBq+~SEPwj$Z13^r4QlEZwo}pvECK!xZ7iT?qX|u;O)RmJUPnJ9mJH$
zXFgTRsW}D<W{FJS9<80cTod(d-teK~*&vO>$1fK5;Fr^79$zNrTllVs3YLrTGix5Y
z<;rVT*67J-w?f_NsZteiAt7HXy{L(@{Y(~&DcY0Uc2RoNla_Lk?32y?Vd+JLZQ4b;
zJy`W?{%9v_!i_=w1qYM`BU_OK@4aES^jFYHrvwGBydA68@jK-%!4cL-ae6d>En#BB
zb8spn1%o?v6YU+?En%qeSOmjONRfaeL9<@2{CHA<m5A-LNGnU9;@aufIJs0stEbv6
zDpZBTqtaBja`GC?Sk-WZN%1E;P@cg}R>cdZt`B&RB<iV<nv#859ayJm<I(YuE)2W8
zZsHy<YJ+%edvj_$3|n<Zm@k`&vJe)3_zd&NVa-vF`x*Ky%HDx$QOUUbA6;EU=~9%<
zyzPgd%bkrQT;rFniqRCvRT(IL{AQfkH&wt(qTYcsT*X1^yqeX73fDRQWP)UNTe(kP
z$-?#<FYLj?Ju**d6wiP2P)J4EwgpLC^Ofpb!#=G?wWlQdo4YpeXGUiQT@7GOq>MRt
z)St5U>1`*Lv5%R}y@3jC*$JF&o+IT!8<wUNNEZurW_yym`yX6gmKJh1>dB)-`=~lZ
zYtYNO*+vq`xHZ|aHes|gZv??flbAc_+!U9dmz()Md4chfv@TcuGRP4o?3o%wQ1XQ-
z<7}jQ+-vE4QOR5t^0s`*vYuA+j0mw0g6z(uq*S?`!|_K4&q$cSH;+ce({~+sPmw)}
zDS0d&W|m%?%&F@3u+XqV)Jz2%LNWPvLi~(`%ds-5kRo=v#Kq$Jhy*R(!3QI#v~Pyx
zE~Rma1NwG0&9f#NuchwPhP32z=CcP)%%42taJ0}6S*>Vl;`1e_q>8R{(0kJBL=0Dw
z!24o~t`klDRsw-$IuENC^{yUsYyWczK&J2yMds$^{_iD_SC+PIMiSS8B@o%?a5QtR
zlEA}$)3=-WZrLGgdk&Qceds{q5>7JbcA<=^2!xQD>cHD<(J<CwzqIpL*{|N-RC2?1
zpM*t|QwX+noOFJ2)9QFM`+`!_d*|U5g8PysNYvG|BbR619-JghayQpVG-eslY9dsB
zFF1JdFojoCR6<UZ2jf%L^l+*268S4gOSc(M!fPDwPz{yMbwtk`ygxfKyb>~C93j6Q
z8*Cv;2eYqEG&b~jXh^gFBvDc-;StTlrQ+7liH{fOJv!W1IJ8PC`?jZ*x$llQYr&vD
z3%oyCy+U%xb4>ix5vQ`;;)RB+mEneRDQC>-r25-5iyQlqdB!jHVCSz{+@Dr~=P#^+
zkTSpRK@{=(?=T<uRnm?&Pd3erk#F@R@8A}aWi8FiPkCrMmL_gGrgMLsy_YuJvUb|3
zlD6`g<uzQ!1Z|mrsurhwNwqLbI@fY;RfZ2z=X+*sPw=t}cZ>CLKVuQq7j@Ph%e0&b
z=NCnbPw{k8Zhsl?Ny_1S;-n_%&weYN|6$M}w_}wWiUFlJ*5L2S-qmRZQnI@sE@MTh
z-==d(&bHbi)H$==?=9s<6P~}*vgjRSU#5LCr4}>yN&HELfMI22A^#(fM72~Q)|n$z
znS%@NRg71XTR!l5i*j}Ov<;kIAoKX)JLKsVHcXcC{1WjVoP#pXyFAKY3L-yb3qAnO
z>Ek5}jK6_jy#1|yjMfq>ZD231?9|7it|s%&;6$_Bl+s3Ver<-f57*~?b;nOzN`8N4
za*&rbu7&GkG>vTeOO31R$`k^LFWPd~-qD2-r%xGXanpal$<p(or*+caQE%Vv;>^g@
zN!&B>lwKl+0d;!3OQ+w9;u>ghYLZ-9U3Uj=wcjA1CBllUQV$$-+e~ZBG2Re+6n5Wd
z^0ub(J@xx{#r<ENAWIuL9`9mw!O}djgK)Hr+x<<IVw-+J?8xH)Cl=no7P=8i&E}fq
z%p=d6IM|4lH%zg#7m@u*KH}e2(fiEPFV(zw{Iu9&-F!5wU2q`GEk*GV)>vW>c0zA6
zcC(X*|G)KGNLkkQ5Q<=Qno>Jer3=ySo|RV}@47jU?y}P0J2n#7qMSgXTBlD@L}9ud
zrmQ4gf0&tWjBNQEynyoUu^hr%ow|-B>cM{BrMg<Lo*72=414BRhV*jMX57^kY~jbt
za4VH;SG#dxjex!}X|hYYsaZlX%%8AjK(JDNUJI|3-27^!(1jr@4izsn{<FMv{>~i7
zT#k9AhOB0`@VA&JIl|QaQ94x}%aXSJ3<~_dWi#*e>(8hAo~nI}z;=^vFi$c>`UHP2
zYjzhblp88&cF;-AJeTjtZaYRisD@AKUFcLKFg;oQK>fW@$MhH1+Swl~hn+UGl=t8u
z^uA{o2hn`I|II;+is-GAD1w^dv+E5qej*pg2lM-Nr#6Ql-klR8)YxC{S5Rd=V)^yW
zpe4h5;;-)&i%fiOeTPX@naGD+qp?8;dMDG{qQkpm)>aDF-P3hWo3gV!cDM4<JGOY@
zlH}a$z|*7g6GpjNiN^4Dn@KN;oD;1)cN#c#*b{j#7abN^vw6Gn&67KhYQYyFP<g2N
zj=`)!zUKM0l7}bMN1s)Bgj&y~i*TZumkSBq;2|j|E=kO!vZv#%;GU~u5^!R%Elihl
z6&NJ>a@Gzi#EW`a&BSszl0CCK|G_5tW^my+GEInsEN<WTeHwjc7S~;S>4o3i4eM(m
zWIHT|fr}&BJrd2{d-t)hxH{^Ex=xJqi)M+Xh*>(IZqQ$~6g;_bGLoS;TJ#lX^X-a5
zw@7C69j>3?Kh&q-NT^`O5I7Da3lKf9PxIDvK)9g83}@2qj>3B);~t|Q(oAAi#7~+^
zE_hkZjai%AjQQ9>AX<^tDH-MwX2Nv28+Ung;abhc2H8?#y6GNE3B9Y^wUj7+^nWj<
ze1xpy0ThAwn!q8c*JMtN*($nxYp=3ycQD#zy}GL9Uv6kv+R_%58#ycWOkb=2RcX0A
zB0?oALS5;z+mbJ>P^`PEL&U5EtIXr_)e;^>2_c95mP=)dQgjup?9~?K7y(!Olf$(%
zs}jlRR-7H|Zq_rBb9d~Xn<zh;aO?ll;BLTFO^9Zc(V+N{!f>)U`k-kv(ObVq#!{wU
z>0-y0J_NH|GtZ#xdp<&!EYcNP_EPaB@A3LUpTg-c-|_hiWPA7E1oWD67bmzee<9S6
zyWftuzKHCbFMWlU=okC4X~ljBJ#zwXq)Ne_&5D{&l?`PkjbZ!}No6hO8<fdZ9rt#l
z?PV3Mj8Js6UwcNipY;9A#<$lKW%MIT$m(Kk7^ZGTA(s!ZGQB+bQu8qlsz($vdmQOk
zPBqPbQ+Savv7qN*U(~hu<S$7Lz1LNqY8S?xmZ6}E94jd#!OP4Wgbyat-kd=2G6ap}
zR(^%mvgI1(^q$+t?2ak9Ytz|U<65Ii(O9$}{oruQx#qq~s=(eux@XJ;`wW5&0<5}2
zANPDhUYC{HFL5wVdWcSv4Ogr%__cHV+C`F@pw*D^l!`~gy<dJDK&|S1+k=sNZGe#2
z{}w5UTb(3VMF_+f)kFz7&Oc-_I;X#>f!DooNO!#}G=wEBl%rhGw!@9|h){%cb9Q?x
zQvb**#}x1D!%p49_?mp4IP8ybw$OW&119_5>*_y^#k=DXBD@MKoAE51Pv^}cIGUrX
zgV0!GkKTjkJ+>_9zgc`C72p$25Rdk<JNn&z65imhtM2^KW7B3Hc|U~wu>X7u+UQ9H
zy<=jooVv8@#=u!lR?~JnmZbXQF9($w9P>$D3O3L$S2qz;8r|sS(=LC8cfWJlDR7Zj
zCp|IPvhu0%SbX~7)Z_bjDvDmS8nCut=B`EN8HMx^?yHeu50htqH=1CG$Icl<CivR9
zs?AR~<7%;8UBJ>HUirif_d|_LPyf~*VzT~rPi{%ZDfOco1zV;1Lr2b?Eg-hbi02Ye
z=S#G%EN)HCepsogc%)&1wgT_WC8<ljK^xH#x1K9qJsaHyoVfMO=@{)Y8o>;PD00q#
z``HOk!)yiK`?ln~+DxmzpHfs7vyOLz{tel+W18M}`W`EW-_7xrmoAP?hMfo#R<3+<
zl<!oiPsANWMv}hs`&GHCH<U}$jan1OjNzgZPA|wy%$E1@^efvwze{a>rjE2&A!6zy
zPjBqV_!^VLi7@Rw7Hf~a5$|6Fl4uN438_?!yo5yas3Uois>q5mwTq~?+&mQr(N0w&
z{_u8sE|#ORr-^bHPbM7Dh;5E(j5)lIqnLF5Pa@EJ><&=;e`78pL~peq2@Wp$98zrP
z)Bh@Tx$twu_f2Oyo;zJ9W@V0K$?Z>{Y}e?CjyZGQNq_lkJ+s^bs+&=>TKa?ARr$4I
z3%!F^B+3u*w_Z5p8B!SC7j0Gf?Di4YxYuvo1@7Z@vb~K>rBEoQE;yLYGSiZvG>;q3
zi_j`wSi-Y;^u3*H--gKLT6X?ci%!|Jsmd549r6=(q~5W7Wh-<pE>R}sQ4Zn7m#JL7
zPB7UvmI&CS<aMpl&CeV#8U9+=_G;Z5uWO&I(jLs%BeqBJ{)HEjP!gqRffLkxg(Zt9
zDZd)LdzbOWrXE(w%(KrJE^dgSikYjmCZRQEqU26F;~F`1$dVfeQ!>$SSJn?Rq<U&b
z@QAJSP0fXCY8)SEN4Xv!ExM^D5!96!&?LwwW=&8*_|e|sOu@9E--o4l1@=*D_7y@8
zbPV0^EFh)$LmD%VJ4sxMJbRO31)rjn_(2>GTgS5$+*B1~Mcnmo9Iu!jvYb!dW3GEd
z<0$UG$nVI9%aR^AK`geG#4H8(bxctnlJMM|KZ!IPd3}l=-)JoLfCOV8u`L6aHP77V
zSDGq5tTMSbzWA@*j&7$e2+VyiYJBJ9N%)kLI(-I>w7}cccQP}{Vz0ZhJB@_&<gs1;
zb62uYgc3cOpH;F!PNtbmVX<mIMuoBCAw0MCn8hAJF!pbBh?g7a5VG{((ZjGQ)59?K
zKyAsxS+FcgSd}&$1!uwULhc(x!Z@tW&4cyhk86r#93e&*8>U(i9Mg0^tU<TW>m9Y~
z9^~v1QF8PD#a>$?veUU0PAtBrmPQd!rxfnwSbFixE7OzTJ{(HMqs6g~l;4m(kir?I
z9aA^8RP_Y;BQQV8J;F1RljVM+<nWD~uO!&zJg&U8d?Yz{xm|6Uwk*Cx<H%(S<0qBG
z{A-0BRDMorulgyOn)@!gmmQe>Mo0C7ujMEQZ$jxZ{jsjDyM}4EF7_>4T3T@+wOw&P
zBi=)V!JmO0uzY^TO5<RWD!$fIJhPHsxJv_%?!B48c?x=!u^4ojN3LLX=bZ-$YTvV>
zy^{hknepS%%mdN>s==k7lw#jME#1IKn60opIwzrPR1$n%_l?u=xWK#v+vTD&H_}DP
zhM7gxdz54B@7j3huO8<4<U{qi?e1fdvrP=oSA{JvVf`MD-(RFV|K=-GY^YD=9?ae|
z=KG7>l6-_Hl%YJZWxDns{VFtz^SHR_TbODwAQsBK-%7qRB=}JH4A@UHK60Pl6zsf8
zUwEKmX(Wfw;buSeA%P#!Eb{uLg+zhDz3d|+g(p1G-Dh<|{b(&q@)o*3wl2ljhNG<P
zpUsBKF&TH*59OKOQBXOl|7hY=+0vAHr+xU9ldZY_?2=mV;`jrNPvnzTkDg(4Dy+s9
zT!!6dIAR~pFBO!{S(k7H`6|z_2=}E*lH-xYz))TIXUkr)Arsj>TnT!wpbsQ|{xeyN
zQR=j51!*=<80XTy_g0?C=O`Z;Z(2#S9@>B7wnVU+d=xwXyL(&-Y_nW?48rQ?-ZC-<
zHMmYnU&yA9u{!8sDa9OndR&u}Oz$c^bJ7wzk)r(Lg|Hk7nZroeV%_G`B*t1oX>Gc<
zo>@|LTdO}aBO}K;`71oJu%h&1#8^xaI)=UKX^CG$c1mi!K3m5kF?WGzlP{>K)ezHv
zcnNtKiNUg!iwjTf7rS!0t;PZGY<lFll%^2*sGh>I`OMNYN)zvjeXCl5$~Fo@6}mWS
z`8z+t8!w*sEm2n`+CyIU2<Z6!MqIi@Nv{G=*C2TRHDPqqvmb3VWUHGRel)B5k4EbB
zH8#m(vgF?wba>t~b+5FnS!Qniw(r#iTE|aPedKe@>f}x9x#`b0FK}z<9lMKr`C5mL
zX%E`=2;8`_f3bk-k+M*dPJB)H5JF$%Tf%_5%E8MSYp+}a9s4N>8q7lINZYCmpB%a3
z_~LXs=H%ErS+e8oysC9r_a`;dvaMy}-K`f)a?g?0s5o9Qm5z5i+4;tvet(ln(Ft<d
zOB2q~A%-#{y3-#l%^L`xFkt&8R_4wcM5jyYi_2b`r<Hif2ulyT3X{LSN@)EBmSz(k
z`6`C7?Y)wTtHYzACAN$r&+hZEl#}<)<^>(ZboQYaHY`5P-@Q9rwDe(!@+gB`+`Y)2
z<^5ysg5OFP4qnt!9Z9qFo}SwW_cyaluFfuIHhv|}WL91x0kVJV{c9hF%zW7!auXgc
zt_OWPfP*oWh0{L6U#Qe=Tfrw#w{qTtt$T!6DDJ<A;W-tqzyMpn_Oj#r{O3<PdnbpL
zri^X6MhX&~o-vWH*U!1$e2V$lOT}96i08NNFL6}Ta#{Gv)CP8%zm0QlaK)&!d&LL4
z1k9hJNY3jcJO&$4ypz)2skZTU+|uC7k*G(qs~2GBMu?5L?UlY(g*)83>-R;&_wn}w
zRpzJn$l>zr5vBc=c*rYlZ|bI@9In;Max!}`<G`k#tbm1JkVvSUa)t*lk6QDrO=@Bq
z^OAv-`-!g&Ym(XO69S)pkYbz|J_`P5NQ_Wkw7+we<gIvt|EY9Ej@nzPK@X0o@o~QX
zUUa|tinXyCCg`%5>t}r5jr$AnXU30@9=v_HjxDpLhwg(_?a1w<0Smt~*ox!reuf>&
z31!A84MV|LW)@?)I)a$O>4TdW+p2%`xG?A?&OUIvkaqD5zpFCl8FOp}&1A(q9AC(C
zu`gu0T2PW2V-P~D;q{&YC5U@T@M^X0YJ|LFJ928E>xCiZ2_vg7k6KbXk3JFap0`Y3
z9?1F_WplhdUrKm_O8TR)?>;1l<sP|Qkhc9}uZ+U}Mer*DJjjKu7${vT)gslNk)iGS
zxUW^t8`s9(<5~mXxZ|8da>=f<G%bw7`sZ~hYew_QSD5L%=Q2=%CGYK@7q$jEbRfj;
z9DA9Wm&8au5Y|lc^h)I!k!S?{TuP?qS2ghibTo7>qO40PPahueeXX7C^uu8NSoElb
zIAc>g^<ma@Q|5}WAKi(i85HiEtj{!h&);JHQuWc1j?caQh>6YX+V3_WpC!+K#9L_)
zKk%ns@az%%p?Ury&=msa1|$i@*7&b;A+N56%&*8P<NC*aN2Ri7m{Z}c6w<juHkz+K
zGv^2vpy!(r+$Hl;GeRG}E|>hcXR~0-XnVFR|AT_3o{wUF8IA4bW6nh58U;r)R5K$x
zJtBL?d7B+cN1YqQbQIKV2&wCHpYo*0#u!hLy4eKEv0a@GLv&j?$U5R&hNjCfy^D)P
zwzOjRhLnnvctl%EFPqy%l^X76ARnshJ;zU>gU{Ng7_ETi=%9GawDP%}u9g;_XJr+f
zF5bKrbN0tQKPDPSG@424`sgRZ@t#1wN|D<K6G&Jj>fB|iQ+(W-D{ijW`rG>QDO^9T
z5$TuD(=jMaHyg1W(rsF7S2`<2SoHW=Q54nnm$y?$m?)*@nB<8zGKHi&zbEadV!ovH
za=%9lbI%veM*RjS%SLuW*12}`n+W=#*7qDS&Y7HQu|mV^b#+C(dPZcvANE+EJ;GWv
z`Y(JHMx`^10YzL>1H%+wU9EVZ&}v8~y*V6lhgyH1!zb$5<w(w<<xH;UTnucls$XiI
zd%NhkzR!@UopXZg)Z5FqIrF5_E6j&9SGo!Moectm6mGh5`!kl@FE-I6H0u-Y>|rz}
zk1{B~J#=G6g0VI@*X8TO9y(?lD$^WxOSK-Jkm#08Wv(CmZ!g!&hOAIY&AzLp9E=um
zC0Y<{ouPTMVpB=@&gZeA=h!uxFrLtn=8gvxzPhMS8$BfmYVqJY!ntBE?wRIsswMly
zzJ)2Hs%lb|mqqI$I!;Yh4RPGGJ-KlsW_b3;Bko=euIbbp(DGoRZBtT|ivE1rNALQ9
zgIEDc`VsPMBJ<u`Vb>|flQVBnIQw1_$s)Vp;@5CGZn8$C_e>F8M4AHZ)X*V6x5H(8
z3vWt9IGG5aARdHA6I!w>EpTpLPd7imxsTDudym{2&mQ4DitoRpEHrPK67p&@rC1Tq
zK8C3SDtj#WUa>6pFBDQfqK4!Eidf83vGEXT@k!t5D)>c*=k+ABVv}kfSseZ<?YD?i
z_&AO{NDT`OV(mPT>0Eo%(_dD}hT`dH2HwE8?3&~BqqxMJn)XRq^y|<8*h}Ie2exT*
zgYq1L4(A*t#`1A)W|OSE=xg<(;%==OA4sMg`xTf2Q^$$Y$(|q1=4&FG;7juH9&51D
zUDYlTe-=uW#lbdaE|O3~rYB*QR3D`?8<Wh`Z!c1?$E^1VThTm!;hu$51Z~Ao#G~FY
zT{sZ)-l+7Mh;d)e%!3S!;<CJ~DhSRJ<(ok?%vA4!5tl0>)?T~bJS0P@;;(XTK|JAg
z)zgHM1Dq)u<wlcEv>S9)w-t}-%9I<kK8dC!xTNB2_;yx?#W{d}oqc_5X(rg|jE3p+
zu{FP##s#&|8SP~~&0}}Q3QiG_oyL?gliWRh?!mxCgrJ^U#R-dihiRMAS-!Njc>;CE
z!1kHPZ!X+gc~ZMBdf`RF;E(>SX+z4$x|uuQSW<i_j>i?dd14XXp8}TOkYq9*)|x(0
zz>sR(&RClLX3&S+cv+EN>bQkcz*o8Xrr{S<Z*q=3759Fe{9@YGPVk!UHEsN`&M>(J
z_<LgK5$p7K?u&uC6^o6+M^(=!KIsmH4Oo7Ts+!ibKdCd%zWl{H(@4;2k@)fo&Fa!r
zF`=34c-+MdYe`#sw#As9*D0GT4}G4kF%?Y&nD4PhdqmCLf0HMWP?o+0LMZSC-^>5u
z=%Tdpr-+SB=hJ3={3lOxBrv}#YZ2%A(R8eX3PGDrMDCZO?DAynq{h*N>vs+tSV@Zp
z50}}KP>Ed@{Z2lRiGSbVV^SsVexqvcGgm`4ZoZ_WPQo-UXN;5avg-;7S>_@y<r8Ky
z8L3;0sodxZR`V-dv*-1>srKO3S})xSk%72QVyS{DkA5C~JE9%S+1rZ8MY=|!K{x8L
zRN^@zBWcXzthJzj%ImW~?Nx@V?*Zm-o4UxNx|g(LKSdh39zmQ;kqotrBLBhomM-Pv
zk3-~DD)M`9YL6Hm#rIb_{4=A{M4cn<bFc@k4&_vA<g@C#*KZTw?6#K*V&gKT5h5?O
zk1?IN-g_)hB0MTuY12DZb?j5V6;^xy?P3$8U)aGmrfzL}hggsKLHbN7)dY!1JRk2S
ze-q`#y3tbs$7H5okVHOMF)#aZeK4zFL+Z>18=>uoMrq=%M)aDYEb+6TMe%2{<^9Uj
zU5{Qp+mBD*?JggkpP)J*6N48n<MU+%v376p$)hpd0t2yuJ?6hhu>CjhBcXieK2aSO
zBh^EyaH-+G0l5>c5kx&b*O8*j{Mt{5BaBHe=#XkaSr_qm(v3d{#_SOt^Z$iemcJzk
zHg;Q?ENT(U0q{ah^hVlCUsiQ3{gPzs=>z!G<2(n8h%LMba_0?R9yD9ID=0B>y?-`i
z$TvDIRbk+nX2!t;0}srndr3OjC~k?8p{Nccsk*`0^cB1)yc27CP`X#ZiTaCxQ$C!Z
zv{{5e!c(D<T>mQGtF~a3h_4j}yqzTHyoIl5N?l+)q1LzFlqh<Q?MLbZ_ya?KqBIx8
zoGF5?E?}_MTwRDU#h&wGaTTZNl~i#`!hSU^c9xojSb<qTOeJmo45hbfh*M6f-79yN
z38Iwq2`ij3dl0`z(8!Jan~dRYWm1rD5!-ZvaVfF0iA1hg-}&4Rc2lO(71HV}l*GJ+
zL9#Jo8>d^uT2b$Ar4#UHwdd!kBHwZ_8uk-L`<{4b)_=^mEO@D0dYUpdvdilgrp~v_
z1MyWNe{QIVy-{4hz)52Nin);^@llEW-nNI(6{1H23LmB#;f0Lk_avvYA8>bxU1z<B
zlSK_qnHM&8v}KqE3UkpVh&=D*if`{4<m8NBzi0;^Y8)Fqd`#%G&iyMfgf3=my#`-s
z)2g0*A*xvB2*;)&loa;h${wK_-(R>e`R)?(1~|c?zGQY9Jjx?$?eA3LzmKlVr;%(r
z(7(<sDJGPmcA58LFZ>~)IU$Nk;(QS;!<a|1Jksz3E-Pegl@mW1KlD<G$lPa~;w=-|
z7*oDC<K`aZ?GdtYqyEAu{hJx$Qfm0}A{l&u)j7JUrL3T6%KRe(7;m<^mLpb%Rg+f&
z+XMN9n@jzAMPM)4#3_20>@R*4qUI^klbz0pxF1Q$V)&3vf<H~#q#qrA?SxlK+9DHc
z1&zSU(34n~+vnyx)~1f25{P-zrd+J3{l@1O&-AX(7go+>pQdg%q-mi{9C&mh?c9y(
zdo?Or4Haow%KJ+PYjnH|t3(BLZ{g*ia=FA3J{TJR*|JHaX6XgbJ5SXcG_r@UC_Pv#
zA-ggVmAN_mTIRg(6Didne{u))YuL4YrgZC|DB^;FlgUm^Q%d5J+*xrGOM^?!swR#W
zrck`+=!6q9H*f^oM}(ZgPMsTrfgzwq;wDZ;ju!SfJ4ZP9)NShwr>e6dZtKV@Gzj3n
zf`JV<_<7-vzPCHLuzlK-6ALmxoKQf-jY9ImQ7G`8rVbb#3?XlTbF|nx5z2)`LPiDr
z_fHKv7z&6}0tZ#WS<tOxt1yIwEjZc<9b^6X`0P&*3?Xh|YH9)wp+culufWkDVeMpZ
zU<3}t0@Gv<=4@vS=K%)Q!^F`Jj<B-@q~lyc4QK9X0_vvVyejzD!Ufcw096=X7{bKD
z%p3>j1N>2f^MU?U;QYW+INKVVI64{GIRZ>GM+2yV0UTinM;O5o#&87G7kB_in8Ohk
z;1nxRBP$rf8ji4mBW%IIfEYN!0geEpIf2nZe{h5g9N`KMvx1zzJ0K8zzX}eoA~Y<F
zapqv|kZ%5Z4t8rfe!d+4cfEgEur2QXPpboXZ*5@a1Q_%;>m=%A1Rd+;=LX;8KqEjm
zoZKK)3(hUuU)*L85{A(HSp|oN!Et5_BT-v3YZE{?LKSCXa|x=*8@O%F2uuj<-2Xk`
zyfr^)I#=MlNaX*E0{g%12MzW?JV<UJF-SNcnh(y4=7sSC$IQnMPAp?Fa5T6=1q0}W
zo<sQPf38>v|9>m6pU=@~ENF+_>I3QrDj3ifg#;aAxw+w3GzO04#Q@QR>VPTyJa8;O
zKb#NC4G6%1@19X`EKn(4ULMehn+H4q$RJnF3$|O(6^lW`dBKOyyu4r(kQ0GA1&Dlb
zEckYs7fg>ANHUlTY>N~yr7aS`=+M+K;MFNF$kt&XJm?Bu{-A*tLv>!TeMEx+&_IoO
zfj&cZexOzW4%iO*-oin`d9YwgU}+(CLTxb6Y`{x=44@dQV?Y8EVh1!nbcM#>S~iG<
zNI)0FS}-OCT({V=Mc593TM`6Kj|TuyD5wiifu{6x<+qxHrWh0o1`3!0=n6uHIQ(yA
zi$E{|KK`wiTSS3L|CEED>$P*;V%ASOwtC_LZ07|Wg{X%F;h*jyEC>iK6jY#lULZ+Z
zQ%3Q@p!=<75L-b9{Ji}C6i5=bga8tRT?GsPRvz#Vk_u2@J6A|5ek**y?)(%ANFaVI
zfUvE5AQ+Hv>=f9RT!73WFE3ymkb%7l|BhThGVyEKk_kvIw#$wz@Bp@g_1{&t*Lqtn
zAldj?Ai3C<JIpVi@c+8KR)V9Totz3}ioxn`)ePV$C}syRLpT_Gt7Zh8&34Tg4mh|4
zGl6q&S<;_o9caxK+!PKrg4;F7_;YW$njL=tgmmkP862q2c5Tbx18WHGV8Fz0eR=@K
zwg5u2jbR0NuwAnTGO=B=frGr)R(o4GQ0=Xn9UQFlRxgk*0y4Y>wTFYn*{*H7EcDLg
z9DuxRKX8O^rF_AlPVlW9C?Ev#SYRc$8t!<#odI!ho}B@=9Ubt7TUe0q0~Tbf?hH9b
zP=o#I+rX}8>!Aw}-tC$zP^ayhJJ8?lng=jkJ2l9!Vzx*9*SbRwfBGs(6nraI`PG?0
z_&=YEqJH)3|NNi@LQE7OA&kI6Lv8~h%Kbm1ae?&WzliAn5fOOWf6;jUM>HfC@ZSF-
zV*VFI5Lp03Bo`l`@fYv*OKkQpQ+^TYe}T|i>~P{=AjIXL?Adnp+rJWmkhL&|qMWUt
z<4}PDDlt1};F`BMv~&0c^UK%>G4M+mP%L*E1YP#lb~tMbL%0i?3(LjB0XN6t?41M<
z2%CQ%aoIVVvBMA`_%e1j0za(rYb$$WQ@Ejlkrfzarw4$!)xCwCtvK`xB&Wp%xPenb
zAwjSQozUSui{w6qM4n>*g)eT7CZ;f8ig{tcBJcbG3yA^RVG92T2E~2QAKZ2ah5}v>
z$lxv*3X;a%Fz{A%H;e}f#C#VX52VYxVUWcC1_L+1IPHd^A)WsnkB0}ddkiF!`;T#v
zJUo!b|JD`-?BDKjQ5b-@8;^${QrzF$@?yaz_IDT<132N`Ff3$sert=wfbXyVfPuz;
zz_1_)-qn^D2^__47|)-4;e{fSU3ge-=pYyL_s{ww!TSC|GZu1YyYcwJ{&W|NkLM40
zLGocBI|lvzI|d&g<U4+c@gpJAwF}1mhg>1~AxFC#5BmrIfS-nZ$8Y_ikdSlS4dVe0
zcQ=d=@@T)e1<Ld%3=K}e?8f7TOv3MNfinI<3pWqsk#^x>AfLDk25sMV!O$oWP3?y9
z@a^U)h|8dN4ZrsXK26`vKgdS>3FH2g79I=;BzE=3!wbGP+yw)o_6rQ>XkcM&;<)vT
zOR5$gCcx%^@WIXwGL~BoPTJNKI4(fJ_Abi_XW)q2GEd+wDjy$=l~qDf683)pAng{6

literal 0
HcmV?d00001


From 57656d89363ffaf3cd22ebdd3fad2ae802f5ffa9 Mon Sep 17 00:00:00 2001
From: Niccolo-Ajroldi <n.ajroldi@gmail.com>
Date: Fri, 11 Oct 2024 17:40:03 +0200
Subject: [PATCH 22/25] fixed :bug in summary_df

---
 scoring/score_submissions.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 4ce1dee16..4cb640515 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -98,10 +98,6 @@ def get_summary_df(workload, workload_df, include_test_split=False):
       lambda x: x['accumulated_submission_time'][int(x['index to target on val'])]
       if x['val target reached'] else np.inf,
       axis=1)
-  summary_df['step to target on val'] = workload_df.apply(
-      lambda x: x['global_step'][int(x['index to target on val'])]
-      if x['val target reached'] else np.inf,
-      axis=1)
 
   # test metrics
   if include_test_split:

From 223b90b976a459eb1a018d968ae6175fce24530f Mon Sep 17 00:00:00 2001
From: Niccolo-Ajroldi <n.ajroldi@gmail.com>
Date: Fri, 11 Oct 2024 18:20:01 +0200
Subject: [PATCH 23/25] fixed :bug in summary_df

---
 scoring/score_submissions.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 4cb640515..79195a063 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -95,7 +95,8 @@ def get_summary_df(workload, workload_df, include_test_split=False):
       if x['val target reached'] else np.nan,
       axis=1)
   summary_df['time to target on val (s)'] = workload_df.apply(
-      lambda x: x['accumulated_submission_time'][int(x['index to target on val'])]
+      lambda x: x['accumulated_submission_time'][int(x[
+          'index to target on val'])]
       if x['val target reached'] else np.inf,
       axis=1)
 

From 076c1a7600eccdd4613daf101fccb806fb06e623 Mon Sep 17 00:00:00 2001
From: Niccolo-Ajroldi <n.ajroldi@gmail.com>
Date: Fri, 11 Oct 2024 18:21:20 +0200
Subject: [PATCH 24/25] fixed :bug in summary_df

---
 scoring/score_submissions.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py
index 79195a063..1fb39d193 100644
--- a/scoring/score_submissions.py
+++ b/scoring/score_submissions.py
@@ -96,8 +96,7 @@ def get_summary_df(workload, workload_df, include_test_split=False):
       axis=1)
   summary_df['time to target on val (s)'] = workload_df.apply(
       lambda x: x['accumulated_submission_time'][int(x[
-          'index to target on val'])]
-      if x['val target reached'] else np.inf,
+          'index to target on val'])] if x['val target reached'] else np.inf,
       axis=1)
 
   # test metrics

From d0fa8889a893e2624c4b56e1527cc51ed11dfcc6 Mon Sep 17 00:00:00 2001
From: Niccolo-Ajroldi <n.ajroldi@gmail.com>
Date: Sat, 12 Oct 2024 18:07:30 +0200
Subject: [PATCH 25/25] removed pdf

---
 .../performance_profile_by_score.pdf            | Bin 18888 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 scoring/scoring_results/performance_profile_by_score.pdf

diff --git a/scoring/scoring_results/performance_profile_by_score.pdf b/scoring/scoring_results/performance_profile_by_score.pdf
deleted file mode 100644
index 4d654f2ba0920a39821072a085332dd8344835a6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18888
zcmch92RxNu_;^~lRzya|jVOhCuj{%-L{|2WWbZw%y&_RYWQ(lq6_TAT*(*_^sH`#~
zq2+&G^!=9ne&0U-&;RrH{hr?QKJRnRdCqg5b)NS<msLqzk`u+n17j_C3D2*Fq2Nfk
zouMV{+&MTx&E4Jvju11z8Ccty!4XOZW+qN>Zh)Wy7Z!$@*cwBKe7`i1w6n#*(OU=z
zO+`aX6C)fPyL~I+hLcjo8Q@IdJlg;z102r8(H4$@Zea*ja|2@wTQfNC&QnJ_BUKX|
z{0bOWToRCC;)a7Gq-_8R7j~{<J6CDA&M)Mk{+s|rH$=Uo37~G9e1wXLlby4p5tt7|
zzX}{7Z(?j=aKX+EjEDsPaie&Va4a_xt^-4efmwm3PH_HhR8d=7J8*}B@P6e4^!O+1
z<V<YMaOQB-&Tx_z)?l)5grqg#fVhc~ov{f-vlGtI#J~pTmNb9&iqq3e$F9HnsxgCd
zzrnZGX%tVS?tbT_T6gnPN(VWyBovJ`LCm)=m*>hKuDu95j-?U1VH!;Pt+4#5+1l+-
zxfKBm-@U(&wtxH*gufP4|Gl#M)OiNIf^SP_f1G1@3I466{kF8vH@S%3$D<rpC+EvQ
z_9HWJ_P)mEW~b$<vGd}U+V%Yi_hL(LoI{RD7he6W6^YSG-1(HA^KrV5BleXeQhZ->
z%)d*eo|FwSvJv~}U0}ecN4CkBOtw}>GDz3F#4vtpd}U;*XiT$u<x%Aeit%7t!*(Ms
z{&yUU3Vq*5FP$%{W70L;a1cz-_SndQJrO*n{$?htq-fz@hTSE^$9miK#|Heut0mEM
zUe1F=PIN4i8Y}kSterO}=El!|zj{a5tCam~b6fr<vED|%x6n6NxAn0(CNhmzcH=MF
zPAkA}T;V9`Vo{k%o&2EpbhTBQ%i!emwP#t>;{|*hT*0cAjEO7V74G6N2DPN`A=Pz!
ztZOATG>K;iiR@I8?6K%<DePxt_|$USwczE?654v93uxvP*~;g<lnbVJ++M+AgL|bO
zr$0BSiFf6&?o~MK8Q0&Ys~S9b)=nkO4eQjJ{Z`7$w&Wyjv|*6?3<J?<#c@)1;;PR|
z-#&FDs?XR?^<^EzOo*hDayF5klbhEYNruHtJ=P3yeW-Z9R{A<c?}>+Hbb}*Ku7jWH
zzTD`2!oj{2>9g*!&UWzW$K`s5^|_g$_UHDWjn`)<*FSiDA6%NVUU&EYXj$p@edR$^
zNJ!L!Y1fal)^W2Asb>mn7GhqXc6&fJ&Dm+Gbi+KhY^C2A8%QH0^Zfmc(v!Y{PcPn|
zr*Xv|Uvo)tvokt_cVDShc`0JDue7{%f_0u<p@D5YxKlX$$Y+gHFLj&^3Ei$ek15Ez
z_xi+8nu$$*v+N7iyEjOx#cR1U;@31zjp;b61kG1Z&2))2isgxQzPJ<=SC%b6cul4#
z<C&`E*L>B5B7wwAo<&2AAu!Aw7-k3zldJI2;M2owEtThbNsrsES)g9Y5w)PIUw=Af
zd_KMIcv#I3{iXXW52($iN4}p=`Mhb@)3fR7;qqyH(t9o$H})oL&S&7sr3>9><yIXx
zQi3z8U()er^#w685`XiXc_1WA$0Jvw<c+Fv@G7OTv>Kv6?Z3vyEs%3#BJEatP+#5z
zVTwoCqwK~vMk|<u*m*)RTNd8w;AVa&`pL6$eaAyBufLp#>Ux8N1=a_09}GRhq(VGc
zbaL_=$(aNr?sJnt(Mv_dZKvMrEIfbhAD;a*GI4@r`b-)NFTE6>by-o?iLCwEHk#CY
zXRhU4hgF#{(&fdfVEH5{NuttdZ#m`BoULtH3O@5mw^qldjd!0BvFXG0Z%PmO<|7I5
zDbq6Px-FeJPSbUmUP?rlpADLCEI39bHYC87I}!hu>x*^}WwE&2+jHK5Rrd&l(gnIc
zK2|OZct;~ln}9f^8tg8UL-&%IMpr+u-QBJ@sg=7;0ZVBuY5H)8ijp(P{8C8njl7Am
z_@QT-$`1osuDDP+%V<snIMEBaXt|NB3_K3a4P|ew6;=J1W+a1J%-P?8zKgIJ|CUiS
z<a*5^7#3nPR#48WhBrOaQ&3J7o_>9%+Y-g8jCndzRQ^Upm%A&@zKv{Yg0kDvQ-n%u
zt-YYUEwHftq4lK@zjRAHPu*r&sUejdG23`t;UuL+(}+}R3xjHKr7Nk)OpbF2UQeIF
zH}a^CgXCYmKCB}1<*w$2edi>#2o$q0lzF~bP;)7|D4_Vk0;6)xi+eg)UM{u!amK^*
z2Riz_yJFjJElb4BSEEiTpQp#0ug+Fo3_V~pcDW*{%IOF_Y{2t<>{o6>>gXCB*Bf|L
zq5aRDg97}^KGcG4&PNwN1--sdYKl#NCp1CXzhOSiz(ATqE{LhI{cvBoG5&!L_A}++
zk8_kY3-Kjqo7>GF)_SNF$K0(t_VH+WbR~MYcs2CM<qGt0<o(kI)`Cu)@~6m8m<bN+
zd*<kq2s)8?TB#j5{;sFmD(HGuDZoaE5|+3=c;CZl>6bTB6-bfhU$xNnu(l-gj?0u*
z&xj++#&wGBv*&vI!@E_7(?kg!USn%3Z@{vO(Zj)ZIr<Ku%L9^!tX5s-q67RJ60a57
zH2N`UU9B>?c3AmSnuOrdr_@xG+FA>qgiIffUb%AZ?PR4&nir{5OPFhgt7ft}#bw5g
z;L>vfDrY;5xMJJ1Il`+n8CRY!(}cVu*Z6tQ5ZMX=P}C5Pl=kHvgzbB%%_Gb5bB_((
zyIPmyx6yu3Kl}~A0{Yh6>f5R0+!Vkx>treT{oaTJbbTe6uw_{e9&6_Re3_WK6YZx7
z0r-@50RToDcr;<?&UWM{k{t311fV~?3ji?Qh+N+4cN^p1CVZf=Hb?Klf4ed5G%ETH
zuuaYWl5;TOmgScjmT`voZD$9U843`}%LzV(Qrh02)IDP<sCzc3O~~X#E<DBT)nzte
z#L?8I9S~kcz~sRC`#nIJWcCV`SdR}`LQzi`aW%Dh9TR@c?9~$U`+b+h`i--{JO}sN
zO;|t^&8@!w8IGmk_j_n~PYkOU+mQ*hA3+MC=>%Fkp!Xm%1b8Dt_&We#FyS+M1%@mo
zi2OuCH*!D#htypFfDwLRyWee$f1A7kjqS;WkKXvFoA~SDRTRGhcBm0wA9v~Q@`UM)
zdff{Wx)iXKHYa3ouT|OPi^1Tl)}pEJbrbgv=N=Qb%sDf`QWOxrUh}o`Q+GN6lM%VF
zZBpGP_9sWZHUjO|=>c|fGKKKegQfn8_c9QZ$ThaP|H*+cUzp2bsExJgOJ$2XlZF$h
z7Hn1HD(Bu%Xn1Q+4zy(k=Izw;!ezBr9d07LUuK5ad~%2rUWj|q=B#Q^-Z7j_wDvOQ
zhdlLy%~Qlg1FrMclDF}M;8N40_W_H2L~FLv>+uNcg|XZv@AOzMJ<)7HwccveBZ>86
z6ITN4$m=B5Z6(%)zbTNIh+wM&!|N@NZY?d4k&Qy%W>?v!3hm~#Q(pq@s&@as*jXF6
z_-OIz`bex4i`te>)-fLG+|rlJjQ{8pxZhD<2NOuYpy$*09XlGihXVfWV23+kvO1p~
zhz?KYy)SyOqw7$I8`kcjt;;v8{!}ZVos3|B9mV^pR!`4&O8kG-&DHx6v#%dKQhjq&
zr*dt0?hTjTdrs;F@BSt!x7jy&`m=BD{8Gjr-=*_iAX+Q>O1&1Jf5qu0R5}Z*0)`5!
zT6W5v_kY5mXLkTD;m<ZduRCq;oSk5?*p|8tjvejZk-~rOx3yggEO`k5)7{|H{|Eqw
zSHM5I4++Fp2RE#?RX0|6VW7=q;>wpw$#qbddjG~JQV<jC8^Y@Avw5tujY>H35HK;o
zIMlDCXqs{~N$p{WDuH?cyA(&X7E84Dk3_}_VuZinrDNJipo5H0%JuM%)J|3laqM2!
zx}AVeG86a8&p*WRD{<<OW@L71?^M-f<MxjYyc5x??oyYHJ1_*Q+fQKNNtNZSW2#m)
zeNHSFeqLs$Ul}eTKbGrIHK^`9x%dwf<;-tLTlm{Zl%P2<ZbyIi0-$hCoylK%`Q-^1
zPT=*aHhF<liea9k4`zyz*({UC!&;XP)Thh~FCD-CeDlWA#-X_S<<BNBzw+8KIbsv0
zKc&{l+ifi8zFQ~xa6C)u?BqJrg<y<bvf-`n+L~GX)>8%nj*TnG3g>;=Q*m#ayB=0Q
zF4KPW*)7w}&%<V%4Mo55Qu5mBTyb2AZr=EFR{rE5dqRa%S9Q*mKcKk}?<+@^;zSK}
zdp@&AE!!$mCE;gL-_Ki$m0?s=w5YH52>MEfUs)f7v>oShS>>`#M%R4wA^Q0Elx(p*
znqhJXXRk3hv3xV=OG!DWjg{8qke%M<w<hZ8Q@#aEkG#Bb9?Ngvx_RuIUy(Y`q*I`%
zn0$YA5MQ0a#ZH^<^Q%f8GWr@EJ>F^#;wIg)94V7sXrkUDpJM8^`U0kUVV(0Hj&-`{
z1w6byedta`4YXF_YnKa>Q5&_Sb>q{Oh+xb~$e6G$O>y<!gK71A=RLiaKdn8w7ALD2
zc!0~GitZdK%Lx9;$o2L<jG)`wzL%YPbv~~ZhkJ*o<HrDbcETSY8O3t-*o#n`F*+qH
z*n4D<ToJ`sf=*q6HR`>;SEeXl^DgoA@a?nZc=NQ*RcL@y>0a&rpm2MEcON_@UA=H0
zC%wL$)2kKOn1AM~mS{E;`}l!|C@pj2wV;G7r;d5M#pjN`jni%}pJ(RD@ao@rUuv3V
z%ZR}jMq_MSNtr(5C=xoxm6<kuygVGg&3yXCvT%<=f!)mRj?`Ht5{>5OwvY!cXY(sv
z8nUJAZZQ_7FnG6oS8bv-!hL(LdjfMw6hE9Ydw%qlcN?Exoo@Yj#gNZTorM5(=pcU`
zW4=GPmVZ#VtH7htAD>rQggsmylR4^sXF^h@y@E0!J3INEoXFO;A4&!AAknCwiIS~+
z0R+4Pk_tZ&q3p=+oB<yOBnti`VIUDMNYRWW5cj<6&qDAqs`rf?+jXx^ULu(f7bZzK
ziromDxtpgcDBq+~SEPwj$Z13^r4QlEZwo}pvECK!xZ7iT?qX|u;O)RmJUPnJ9mJH$
zXFgTRsW}D<W{FJS9<80cTod(d-teK~*&vO>$1fK5;Fr^79$zNrTllVs3YLrTGix5Y
z<;rVT*67J-w?f_NsZteiAt7HXy{L(@{Y(~&DcY0Uc2RoNla_Lk?32y?Vd+JLZQ4b;
zJy`W?{%9v_!i_=w1qYM`BU_OK@4aES^jFYHrvwGBydA68@jK-%!4cL-ae6d>En#BB
zb8spn1%o?v6YU+?En%qeSOmjONRfaeL9<@2{CHA<m5A-LNGnU9;@aufIJs0stEbv6
zDpZBTqtaBja`GC?Sk-WZN%1E;P@cg}R>cdZt`B&RB<iV<nv#859ayJm<I(YuE)2W8
zZsHy<YJ+%edvj_$3|n<Zm@k`&vJe)3_zd&NVa-vF`x*Ky%HDx$QOUUbA6;EU=~9%<
zyzPgd%bkrQT;rFniqRCvRT(IL{AQfkH&wt(qTYcsT*X1^yqeX73fDRQWP)UNTe(kP
z$-?#<FYLj?Ju**d6wiP2P)J4EwgpLC^Ofpb!#=G?wWlQdo4YpeXGUiQT@7GOq>MRt
z)St5U>1`*Lv5%R}y@3jC*$JF&o+IT!8<wUNNEZurW_yym`yX6gmKJh1>dB)-`=~lZ
zYtYNO*+vq`xHZ|aHes|gZv??flbAc_+!U9dmz()Md4chfv@TcuGRP4o?3o%wQ1XQ-
z<7}jQ+-vE4QOR5t^0s`*vYuA+j0mw0g6z(uq*S?`!|_K4&q$cSH;+ce({~+sPmw)}
zDS0d&W|m%?%&F@3u+XqV)Jz2%LNWPvLi~(`%ds-5kRo=v#Kq$Jhy*R(!3QI#v~Pyx
zE~Rma1NwG0&9f#NuchwPhP32z=CcP)%%42taJ0}6S*>Vl;`1e_q>8R{(0kJBL=0Dw
z!24o~t`klDRsw-$IuENC^{yUsYyWczK&J2yMds$^{_iD_SC+PIMiSS8B@o%?a5QtR
zlEA}$)3=-WZrLGgdk&Qceds{q5>7JbcA<=^2!xQD>cHD<(J<CwzqIpL*{|N-RC2?1
zpM*t|QwX+noOFJ2)9QFM`+`!_d*|U5g8PysNYvG|BbR619-JghayQpVG-eslY9dsB
zFF1JdFojoCR6<UZ2jf%L^l+*268S4gOSc(M!fPDwPz{yMbwtk`ygxfKyb>~C93j6Q
z8*Cv;2eYqEG&b~jXh^gFBvDc-;StTlrQ+7liH{fOJv!W1IJ8PC`?jZ*x$llQYr&vD
z3%oyCy+U%xb4>ix5vQ`;;)RB+mEneRDQC>-r25-5iyQlqdB!jHVCSz{+@Dr~=P#^+
zkTSpRK@{=(?=T<uRnm?&Pd3erk#F@R@8A}aWi8FiPkCrMmL_gGrgMLsy_YuJvUb|3
zlD6`g<uzQ!1Z|mrsurhwNwqLbI@fY;RfZ2z=X+*sPw=t}cZ>CLKVuQq7j@Ph%e0&b
z=NCnbPw{k8Zhsl?Ny_1S;-n_%&weYN|6$M}w_}wWiUFlJ*5L2S-qmRZQnI@sE@MTh
z-==d(&bHbi)H$==?=9s<6P~}*vgjRSU#5LCr4}>yN&HELfMI22A^#(fM72~Q)|n$z
znS%@NRg71XTR!l5i*j}Ov<;kIAoKX)JLKsVHcXcC{1WjVoP#pXyFAKY3L-yb3qAnO
z>Ek5}jK6_jy#1|yjMfq>ZD231?9|7it|s%&;6$_Bl+s3Ver<-f57*~?b;nOzN`8N4
za*&rbu7&GkG>vTeOO31R$`k^LFWPd~-qD2-r%xGXanpal$<p(or*+caQE%Vv;>^g@
zN!&B>lwKl+0d;!3OQ+w9;u>ghYLZ-9U3Uj=wcjA1CBllUQV$$-+e~ZBG2Re+6n5Wd
z^0ub(J@xx{#r<ENAWIuL9`9mw!O}djgK)Hr+x<<IVw-+J?8xH)Cl=no7P=8i&E}fq
z%p=d6IM|4lH%zg#7m@u*KH}e2(fiEPFV(zw{Iu9&-F!5wU2q`GEk*GV)>vW>c0zA6
zcC(X*|G)KGNLkkQ5Q<=Qno>Jer3=ySo|RV}@47jU?y}P0J2n#7qMSgXTBlD@L}9ud
zrmQ4gf0&tWjBNQEynyoUu^hr%ow|-B>cM{BrMg<Lo*72=414BRhV*jMX57^kY~jbt
za4VH;SG#dxjex!}X|hYYsaZlX%%8AjK(JDNUJI|3-27^!(1jr@4izsn{<FMv{>~i7
zT#k9AhOB0`@VA&JIl|QaQ94x}%aXSJ3<~_dWi#*e>(8hAo~nI}z;=^vFi$c>`UHP2
zYjzhblp88&cF;-AJeTjtZaYRisD@AKUFcLKFg;oQK>fW@$MhH1+Swl~hn+UGl=t8u
z^uA{o2hn`I|II;+is-GAD1w^dv+E5qej*pg2lM-Nr#6Ql-klR8)YxC{S5Rd=V)^yW
zpe4h5;;-)&i%fiOeTPX@naGD+qp?8;dMDG{qQkpm)>aDF-P3hWo3gV!cDM4<JGOY@
zlH}a$z|*7g6GpjNiN^4Dn@KN;oD;1)cN#c#*b{j#7abN^vw6Gn&67KhYQYyFP<g2N
zj=`)!zUKM0l7}bMN1s)Bgj&y~i*TZumkSBq;2|j|E=kO!vZv#%;GU~u5^!R%Elihl
z6&NJ>a@Gzi#EW`a&BSszl0CCK|G_5tW^my+GEInsEN<WTeHwjc7S~;S>4o3i4eM(m
zWIHT|fr}&BJrd2{d-t)hxH{^Ex=xJqi)M+Xh*>(IZqQ$~6g;_bGLoS;TJ#lX^X-a5
zw@7C69j>3?Kh&q-NT^`O5I7Da3lKf9PxIDvK)9g83}@2qj>3B);~t|Q(oAAi#7~+^
zE_hkZjai%AjQQ9>AX<^tDH-MwX2Nv28+Ung;abhc2H8?#y6GNE3B9Y^wUj7+^nWj<
ze1xpy0ThAwn!q8c*JMtN*($nxYp=3ycQD#zy}GL9Uv6kv+R_%58#ycWOkb=2RcX0A
zB0?oALS5;z+mbJ>P^`PEL&U5EtIXr_)e;^>2_c95mP=)dQgjup?9~?K7y(!Olf$(%
zs}jlRR-7H|Zq_rBb9d~Xn<zh;aO?ll;BLTFO^9Zc(V+N{!f>)U`k-kv(ObVq#!{wU
z>0-y0J_NH|GtZ#xdp<&!EYcNP_EPaB@A3LUpTg-c-|_hiWPA7E1oWD67bmzee<9S6
zyWftuzKHCbFMWlU=okC4X~ljBJ#zwXq)Ne_&5D{&l?`PkjbZ!}No6hO8<fdZ9rt#l
z?PV3Mj8Js6UwcNipY;9A#<$lKW%MIT$m(Kk7^ZGTA(s!ZGQB+bQu8qlsz($vdmQOk
zPBqPbQ+Savv7qN*U(~hu<S$7Lz1LNqY8S?xmZ6}E94jd#!OP4Wgbyat-kd=2G6ap}
zR(^%mvgI1(^q$+t?2ak9Ytz|U<65Ii(O9$}{oruQx#qq~s=(eux@XJ;`wW5&0<5}2
zANPDhUYC{HFL5wVdWcSv4Ogr%__cHV+C`F@pw*D^l!`~gy<dJDK&|S1+k=sNZGe#2
z{}w5UTb(3VMF_+f)kFz7&Oc-_I;X#>f!DooNO!#}G=wEBl%rhGw!@9|h){%cb9Q?x
zQvb**#}x1D!%p49_?mp4IP8ybw$OW&119_5>*_y^#k=DXBD@MKoAE51Pv^}cIGUrX
zgV0!GkKTjkJ+>_9zgc`C72p$25Rdk<JNn&z65imhtM2^KW7B3Hc|U~wu>X7u+UQ9H
zy<=jooVv8@#=u!lR?~JnmZbXQF9($w9P>$D3O3L$S2qz;8r|sS(=LC8cfWJlDR7Zj
zCp|IPvhu0%SbX~7)Z_bjDvDmS8nCut=B`EN8HMx^?yHeu50htqH=1CG$Icl<CivR9
zs?AR~<7%;8UBJ>HUirif_d|_LPyf~*VzT~rPi{%ZDfOco1zV;1Lr2b?Eg-hbi02Ye
z=S#G%EN)HCepsogc%)&1wgT_WC8<ljK^xH#x1K9qJsaHyoVfMO=@{)Y8o>;PD00q#
z``HOk!)yiK`?ln~+DxmzpHfs7vyOLz{tel+W18M}`W`EW-_7xrmoAP?hMfo#R<3+<
zl<!oiPsANWMv}hs`&GHCH<U}$jan1OjNzgZPA|wy%$E1@^efvwze{a>rjE2&A!6zy
zPjBqV_!^VLi7@Rw7Hf~a5$|6Fl4uN438_?!yo5yas3Uois>q5mwTq~?+&mQr(N0w&
z{_u8sE|#ORr-^bHPbM7Dh;5E(j5)lIqnLF5Pa@EJ><&=;e`78pL~peq2@Wp$98zrP
z)Bh@Tx$twu_f2Oyo;zJ9W@V0K$?Z>{Y}e?CjyZGQNq_lkJ+s^bs+&=>TKa?ARr$4I
z3%!F^B+3u*w_Z5p8B!SC7j0Gf?Di4YxYuvo1@7Z@vb~K>rBEoQE;yLYGSiZvG>;q3
zi_j`wSi-Y;^u3*H--gKLT6X?ci%!|Jsmd549r6=(q~5W7Wh-<pE>R}sQ4Zn7m#JL7
zPB7UvmI&CS<aMpl&CeV#8U9+=_G;Z5uWO&I(jLs%BeqBJ{)HEjP!gqRffLkxg(Zt9
zDZd)LdzbOWrXE(w%(KrJE^dgSikYjmCZRQEqU26F;~F`1$dVfeQ!>$SSJn?Rq<U&b
z@QAJSP0fXCY8)SEN4Xv!ExM^D5!96!&?LwwW=&8*_|e|sOu@9E--o4l1@=*D_7y@8
zbPV0^EFh)$LmD%VJ4sxMJbRO31)rjn_(2>GTgS5$+*B1~Mcnmo9Iu!jvYb!dW3GEd
z<0$UG$nVI9%aR^AK`geG#4H8(bxctnlJMM|KZ!IPd3}l=-)JoLfCOV8u`L6aHP77V
zSDGq5tTMSbzWA@*j&7$e2+VyiYJBJ9N%)kLI(-I>w7}cccQP}{Vz0ZhJB@_&<gs1;
zb62uYgc3cOpH;F!PNtbmVX<mIMuoBCAw0MCn8hAJF!pbBh?g7a5VG{((ZjGQ)59?K
zKyAsxS+FcgSd}&$1!uwULhc(x!Z@tW&4cyhk86r#93e&*8>U(i9Mg0^tU<TW>m9Y~
z9^~v1QF8PD#a>$?veUU0PAtBrmPQd!rxfnwSbFixE7OzTJ{(HMqs6g~l;4m(kir?I
z9aA^8RP_Y;BQQV8J;F1RljVM+<nWD~uO!&zJg&U8d?Yz{xm|6Uwk*Cx<H%(S<0qBG
z{A-0BRDMorulgyOn)@!gmmQe>Mo0C7ujMEQZ$jxZ{jsjDyM}4EF7_>4T3T@+wOw&P
zBi=)V!JmO0uzY^TO5<RWD!$fIJhPHsxJv_%?!B48c?x=!u^4ojN3LLX=bZ-$YTvV>
zy^{hknepS%%mdN>s==k7lw#jME#1IKn60opIwzrPR1$n%_l?u=xWK#v+vTD&H_}DP
zhM7gxdz54B@7j3huO8<4<U{qi?e1fdvrP=oSA{JvVf`MD-(RFV|K=-GY^YD=9?ae|
z=KG7>l6-_Hl%YJZWxDns{VFtz^SHR_TbODwAQsBK-%7qRB=}JH4A@UHK60Pl6zsf8
zUwEKmX(Wfw;buSeA%P#!Eb{uLg+zhDz3d|+g(p1G-Dh<|{b(&q@)o*3wl2ljhNG<P
zpUsBKF&TH*59OKOQBXOl|7hY=+0vAHr+xU9ldZY_?2=mV;`jrNPvnzTkDg(4Dy+s9
zT!!6dIAR~pFBO!{S(k7H`6|z_2=}E*lH-xYz))TIXUkr)Arsj>TnT!wpbsQ|{xeyN
zQR=j51!*=<80XTy_g0?C=O`Z;Z(2#S9@>B7wnVU+d=xwXyL(&-Y_nW?48rQ?-ZC-<
zHMmYnU&yA9u{!8sDa9OndR&u}Oz$c^bJ7wzk)r(Lg|Hk7nZroeV%_G`B*t1oX>Gc<
zo>@|LTdO}aBO}K;`71oJu%h&1#8^xaI)=UKX^CG$c1mi!K3m5kF?WGzlP{>K)ezHv
zcnNtKiNUg!iwjTf7rS!0t;PZGY<lFll%^2*sGh>I`OMNYN)zvjeXCl5$~Fo@6}mWS
z`8z+t8!w*sEm2n`+CyIU2<Z6!MqIi@Nv{G=*C2TRHDPqqvmb3VWUHGRel)B5k4EbB
zH8#m(vgF?wba>t~b+5FnS!Qniw(r#iTE|aPedKe@>f}x9x#`b0FK}z<9lMKr`C5mL
zX%E`=2;8`_f3bk-k+M*dPJB)H5JF$%Tf%_5%E8MSYp+}a9s4N>8q7lINZYCmpB%a3
z_~LXs=H%ErS+e8oysC9r_a`;dvaMy}-K`f)a?g?0s5o9Qm5z5i+4;tvet(ln(Ft<d
zOB2q~A%-#{y3-#l%^L`xFkt&8R_4wcM5jyYi_2b`r<Hif2ulyT3X{LSN@)EBmSz(k
z`6`C7?Y)wTtHYzACAN$r&+hZEl#}<)<^>(ZboQYaHY`5P-@Q9rwDe(!@+gB`+`Y)2
z<^5ysg5OFP4qnt!9Z9qFo}SwW_cyaluFfuIHhv|}WL91x0kVJV{c9hF%zW7!auXgc
zt_OWPfP*oWh0{L6U#Qe=Tfrw#w{qTtt$T!6DDJ<A;W-tqzyMpn_Oj#r{O3<PdnbpL
zri^X6MhX&~o-vWH*U!1$e2V$lOT}96i08NNFL6}Ta#{Gv)CP8%zm0QlaK)&!d&LL4
z1k9hJNY3jcJO&$4ypz)2skZTU+|uC7k*G(qs~2GBMu?5L?UlY(g*)83>-R;&_wn}w
zRpzJn$l>zr5vBc=c*rYlZ|bI@9In;Max!}`<G`k#tbm1JkVvSUa)t*lk6QDrO=@Bq
z^OAv-`-!g&Ym(XO69S)pkYbz|J_`P5NQ_Wkw7+we<gIvt|EY9Ej@nzPK@X0o@o~QX
zUUa|tinXyCCg`%5>t}r5jr$AnXU30@9=v_HjxDpLhwg(_?a1w<0Smt~*ox!reuf>&
z31!A84MV|LW)@?)I)a$O>4TdW+p2%`xG?A?&OUIvkaqD5zpFCl8FOp}&1A(q9AC(C
zu`gu0T2PW2V-P~D;q{&YC5U@T@M^X0YJ|LFJ928E>xCiZ2_vg7k6KbXk3JFap0`Y3
z9?1F_WplhdUrKm_O8TR)?>;1l<sP|Qkhc9}uZ+U}Mer*DJjjKu7${vT)gslNk)iGS
zxUW^t8`s9(<5~mXxZ|8da>=f<G%bw7`sZ~hYew_QSD5L%=Q2=%CGYK@7q$jEbRfj;
z9DA9Wm&8au5Y|lc^h)I!k!S?{TuP?qS2ghibTo7>qO40PPahueeXX7C^uu8NSoElb
zIAc>g^<ma@Q|5}WAKi(i85HiEtj{!h&);JHQuWc1j?caQh>6YX+V3_WpC!+K#9L_)
zKk%ns@az%%p?Ury&=msa1|$i@*7&b;A+N56%&*8P<NC*aN2Ri7m{Z}c6w<juHkz+K
zGv^2vpy!(r+$Hl;GeRG}E|>hcXR~0-XnVFR|AT_3o{wUF8IA4bW6nh58U;r)R5K$x
zJtBL?d7B+cN1YqQbQIKV2&wCHpYo*0#u!hLy4eKEv0a@GLv&j?$U5R&hNjCfy^D)P
zwzOjRhLnnvctl%EFPqy%l^X76ARnshJ;zU>gU{Ng7_ETi=%9GawDP%}u9g;_XJr+f
zF5bKrbN0tQKPDPSG@424`sgRZ@t#1wN|D<K6G&Jj>fB|iQ+(W-D{ijW`rG>QDO^9T
z5$TuD(=jMaHyg1W(rsF7S2`<2SoHW=Q54nnm$y?$m?)*@nB<8zGKHi&zbEadV!ovH
za=%9lbI%veM*RjS%SLuW*12}`n+W=#*7qDS&Y7HQu|mV^b#+C(dPZcvANE+EJ;GWv
z`Y(JHMx`^10YzL>1H%+wU9EVZ&}v8~y*V6lhgyH1!zb$5<w(w<<xH;UTnucls$XiI
zd%NhkzR!@UopXZg)Z5FqIrF5_E6j&9SGo!Moectm6mGh5`!kl@FE-I6H0u-Y>|rz}
zk1{B~J#=G6g0VI@*X8TO9y(?lD$^WxOSK-Jkm#08Wv(CmZ!g!&hOAIY&AzLp9E=um
zC0Y<{ouPTMVpB=@&gZeA=h!uxFrLtn=8gvxzPhMS8$BfmYVqJY!ntBE?wRIsswMly
zzJ)2Hs%lb|mqqI$I!;Yh4RPGGJ-KlsW_b3;Bko=euIbbp(DGoRZBtT|ivE1rNALQ9
zgIEDc`VsPMBJ<u`Vb>|flQVBnIQw1_$s)Vp;@5CGZn8$C_e>F8M4AHZ)X*V6x5H(8
z3vWt9IGG5aARdHA6I!w>EpTpLPd7imxsTDudym{2&mQ4DitoRpEHrPK67p&@rC1Tq
zK8C3SDtj#WUa>6pFBDQfqK4!Eidf83vGEXT@k!t5D)>c*=k+ABVv}kfSseZ<?YD?i
z_&AO{NDT`OV(mPT>0Eo%(_dD}hT`dH2HwE8?3&~BqqxMJn)XRq^y|<8*h}Ie2exT*
zgYq1L4(A*t#`1A)W|OSE=xg<(;%==OA4sMg`xTf2Q^$$Y$(|q1=4&FG;7juH9&51D
zUDYlTe-=uW#lbdaE|O3~rYB*QR3D`?8<Wh`Z!c1?$E^1VThTm!;hu$51Z~Ao#G~FY
zT{sZ)-l+7Mh;d)e%!3S!;<CJ~DhSRJ<(ok?%vA4!5tl0>)?T~bJS0P@;;(XTK|JAg
z)zgHM1Dq)u<wlcEv>S9)w-t}-%9I<kK8dC!xTNB2_;yx?#W{d}oqc_5X(rg|jE3p+
zu{FP##s#&|8SP~~&0}}Q3QiG_oyL?gliWRh?!mxCgrJ^U#R-dihiRMAS-!Njc>;CE
z!1kHPZ!X+gc~ZMBdf`RF;E(>SX+z4$x|uuQSW<i_j>i?dd14XXp8}TOkYq9*)|x(0
zz>sR(&RClLX3&S+cv+EN>bQkcz*o8Xrr{S<Z*q=3759Fe{9@YGPVk!UHEsN`&M>(J
z_<LgK5$p7K?u&uC6^o6+M^(=!KIsmH4Oo7Ts+!ibKdCd%zWl{H(@4;2k@)fo&Fa!r
zF`=34c-+MdYe`#sw#As9*D0GT4}G4kF%?Y&nD4PhdqmCLf0HMWP?o+0LMZSC-^>5u
z=%Tdpr-+SB=hJ3={3lOxBrv}#YZ2%A(R8eX3PGDrMDCZO?DAynq{h*N>vs+tSV@Zp
z50}}KP>Ed@{Z2lRiGSbVV^SsVexqvcGgm`4ZoZ_WPQo-UXN;5avg-;7S>_@y<r8Ky
z8L3;0sodxZR`V-dv*-1>srKO3S})xSk%72QVyS{DkA5C~JE9%S+1rZ8MY=|!K{x8L
zRN^@zBWcXzthJzj%ImW~?Nx@V?*Zm-o4UxNx|g(LKSdh39zmQ;kqotrBLBhomM-Pv
zk3-~DD)M`9YL6Hm#rIb_{4=A{M4cn<bFc@k4&_vA<g@C#*KZTw?6#K*V&gKT5h5?O
zk1?IN-g_)hB0MTuY12DZb?j5V6;^xy?P3$8U)aGmrfzL}hggsKLHbN7)dY!1JRk2S
ze-q`#y3tbs$7H5okVHOMF)#aZeK4zFL+Z>18=>uoMrq=%M)aDYEb+6TMe%2{<^9Uj
zU5{Qp+mBD*?JggkpP)J*6N48n<MU+%v376p$)hpd0t2yuJ?6hhu>CjhBcXieK2aSO
zBh^EyaH-+G0l5>c5kx&b*O8*j{Mt{5BaBHe=#XkaSr_qm(v3d{#_SOt^Z$iemcJzk
zHg;Q?ENT(U0q{ah^hVlCUsiQ3{gPzs=>z!G<2(n8h%LMba_0?R9yD9ID=0B>y?-`i
z$TvDIRbk+nX2!t;0}srndr3OjC~k?8p{Nccsk*`0^cB1)yc27CP`X#ZiTaCxQ$C!Z
zv{{5e!c(D<T>mQGtF~a3h_4j}yqzTHyoIl5N?l+)q1LzFlqh<Q?MLbZ_ya?KqBIx8
zoGF5?E?}_MTwRDU#h&wGaTTZNl~i#`!hSU^c9xojSb<qTOeJmo45hbfh*M6f-79yN
z38Iwq2`ij3dl0`z(8!Jan~dRYWm1rD5!-ZvaVfF0iA1hg-}&4Rc2lO(71HV}l*GJ+
zL9#Jo8>d^uT2b$Ar4#UHwdd!kBHwZ_8uk-L`<{4b)_=^mEO@D0dYUpdvdilgrp~v_
z1MyWNe{QIVy-{4hz)52Nin);^@llEW-nNI(6{1H23LmB#;f0Lk_avvYA8>bxU1z<B
zlSK_qnHM&8v}KqE3UkpVh&=D*if`{4<m8NBzi0;^Y8)Fqd`#%G&iyMfgf3=my#`-s
z)2g0*A*xvB2*;)&loa;h${wK_-(R>e`R)?(1~|c?zGQY9Jjx?$?eA3LzmKlVr;%(r
z(7(<sDJGPmcA58LFZ>~)IU$Nk;(QS;!<a|1Jksz3E-Pegl@mW1KlD<G$lPa~;w=-|
z7*oDC<K`aZ?GdtYqyEAu{hJx$Qfm0}A{l&u)j7JUrL3T6%KRe(7;m<^mLpb%Rg+f&
z+XMN9n@jzAMPM)4#3_20>@R*4qUI^klbz0pxF1Q$V)&3vf<H~#q#qrA?SxlK+9DHc
z1&zSU(34n~+vnyx)~1f25{P-zrd+J3{l@1O&-AX(7go+>pQdg%q-mi{9C&mh?c9y(
zdo?Or4Haow%KJ+PYjnH|t3(BLZ{g*ia=FA3J{TJR*|JHaX6XgbJ5SXcG_r@UC_Pv#
zA-ggVmAN_mTIRg(6Didne{u))YuL4YrgZC|DB^;FlgUm^Q%d5J+*xrGOM^?!swR#W
zrck`+=!6q9H*f^oM}(ZgPMsTrfgzwq;wDZ;ju!SfJ4ZP9)NShwr>e6dZtKV@Gzj3n
zf`JV<_<7-vzPCHLuzlK-6ALmxoKQf-jY9ImQ7G`8rVbb#3?XlTbF|nx5z2)`LPiDr
z_fHKv7z&6}0tZ#WS<tOxt1yIwEjZc<9b^6X`0P&*3?Xh|YH9)wp+culufWkDVeMpZ
zU<3}t0@Gv<=4@vS=K%)Q!^F`Jj<B-@q~lyc4QK9X0_vvVyejzD!Ufcw096=X7{bKD
z%p3>j1N>2f^MU?U;QYW+INKVVI64{GIRZ>GM+2yV0UTinM;O5o#&87G7kB_in8Ohk
z;1nxRBP$rf8ji4mBW%IIfEYN!0geEpIf2nZe{h5g9N`KMvx1zzJ0K8zzX}eoA~Y<F
zapqv|kZ%5Z4t8rfe!d+4cfEgEur2QXPpboXZ*5@a1Q_%;>m=%A1Rd+;=LX;8KqEjm
zoZKK)3(hUuU)*L85{A(HSp|oN!Et5_BT-v3YZE{?LKSCXa|x=*8@O%F2uuj<-2Xk`
zyfr^)I#=MlNaX*E0{g%12MzW?JV<UJF-SNcnh(y4=7sSC$IQnMPAp?Fa5T6=1q0}W
zo<sQPf38>v|9>m6pU=@~ENF+_>I3QrDj3ifg#;aAxw+w3GzO04#Q@QR>VPTyJa8;O
zKb#NC4G6%1@19X`EKn(4ULMehn+H4q$RJnF3$|O(6^lW`dBKOyyu4r(kQ0GA1&Dlb
zEckYs7fg>ANHUlTY>N~yr7aS`=+M+K;MFNF$kt&XJm?Bu{-A*tLv>!TeMEx+&_IoO
zfj&cZexOzW4%iO*-oin`d9YwgU}+(CLTxb6Y`{x=44@dQV?Y8EVh1!nbcM#>S~iG<
zNI)0FS}-OCT({V=Mc593TM`6Kj|TuyD5wiifu{6x<+qxHrWh0o1`3!0=n6uHIQ(yA
zi$E{|KK`wiTSS3L|CEED>$P*;V%ASOwtC_LZ07|Wg{X%F;h*jyEC>iK6jY#lULZ+Z
zQ%3Q@p!=<75L-b9{Ji}C6i5=bga8tRT?GsPRvz#Vk_u2@J6A|5ek**y?)(%ANFaVI
zfUvE5AQ+Hv>=f9RT!73WFE3ymkb%7l|BhThGVyEKk_kvIw#$wz@Bp@g_1{&t*Lqtn
zAldj?Ai3C<JIpVi@c+8KR)V9Totz3}ioxn`)ePV$C}syRLpT_Gt7Zh8&34Tg4mh|4
zGl6q&S<;_o9caxK+!PKrg4;F7_;YW$njL=tgmmkP862q2c5Tbx18WHGV8Fz0eR=@K
zwg5u2jbR0NuwAnTGO=B=frGr)R(o4GQ0=Xn9UQFlRxgk*0y4Y>wTFYn*{*H7EcDLg
z9DuxRKX8O^rF_AlPVlW9C?Ev#SYRc$8t!<#odI!ho}B@=9Ubt7TUe0q0~Tbf?hH9b
zP=o#I+rX}8>!Aw}-tC$zP^ayhJJ8?lng=jkJ2l9!Vzx*9*SbRwfBGs(6nraI`PG?0
z_&=YEqJH)3|NNi@LQE7OA&kI6Lv8~h%Kbm1ae?&WzliAn5fOOWf6;jUM>HfC@ZSF-
zV*VFI5Lp03Bo`l`@fYv*OKkQpQ+^TYe}T|i>~P{=AjIXL?Adnp+rJWmkhL&|qMWUt
z<4}PDDlt1};F`BMv~&0c^UK%>G4M+mP%L*E1YP#lb~tMbL%0i?3(LjB0XN6t?41M<
z2%CQ%aoIVVvBMA`_%e1j0za(rYb$$WQ@Ejlkrfzarw4$!)xCwCtvK`xB&Wp%xPenb
zAwjSQozUSui{w6qM4n>*g)eT7CZ;f8ig{tcBJcbG3yA^RVG92T2E~2QAKZ2ah5}v>
z$lxv*3X;a%Fz{A%H;e}f#C#VX52VYxVUWcC1_L+1IPHd^A)WsnkB0}ddkiF!`;T#v
zJUo!b|JD`-?BDKjQ5b-@8;^${QrzF$@?yaz_IDT<132N`Ff3$sert=wfbXyVfPuz;
zz_1_)-qn^D2^__47|)-4;e{fSU3ge-=pYyL_s{ww!TSC|GZu1YyYcwJ{&W|NkLM40
zLGocBI|lvzI|d&g<U4+c@gpJAwF}1mhg>1~AxFC#5BmrIfS-nZ$8Y_ikdSlS4dVe0
zcQ=d=@@T)e1<Ld%3=K}e?8f7TOv3MNfinI<3pWqsk#^x>AfLDk25sMV!O$oWP3?y9
z@a^U)h|8dN4ZrsXK26`vKgdS>3FH2g79I=;BzE=3!wbGP+yw)o_6rQ>XkcM&;<)vT
zOR5$gCcx%^@WIXwGL~BoPTJNKI4(fJ_Abi_XW)q2GEd+wDjy$=l~qDf683)pAng{6