From c324bdf0970b511608bde09524ab69cf556922c7 Mon Sep 17 00:00:00 2001
From: harvard-edge <khoshnevis.naeem@gmail.com>
Date: Thu, 23 Jan 2025 04:05:09 +0000
Subject: [PATCH] Push dev branch build

---
 .../data_engineering/data_engineering.html    | 111 +++++++++---------
 docs/index.html                               |   4 +-
 docs/search.json                              |   8 +-
 3 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/docs/contents/core/data_engineering/data_engineering.html b/docs/contents/core/data_engineering/data_engineering.html
index 7832aabc..a91c474a 100644
--- a/docs/contents/core/data_engineering/data_engineering.html
+++ b/docs/contents/core/data_engineering/data_engineering.html
@@ -104,6 +104,35 @@
 <script async="" src="https://www.googletagmanager.com/gtag/js?id=G-M21L0CBCVN"></script>
 <script type="module" src="../../../scripts/ai_menu/dist/bundle.js" defer=""></script>
 
+  <script src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
+  <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>
+
+<script type="text/javascript">
+const typesetMath = (el) => {
+  if (window.MathJax) {
+    // MathJax Typeset
+    window.MathJax.typeset([el]);
+  } else if (window.katex) {
+    // KaTeX Render
+    var mathElements = el.getElementsByClassName("math");
+    var macros = [];
+    for (var i = 0; i < mathElements.length; i++) {
+      var texText = mathElements[i].firstChild;
+      if (mathElements[i].tagName == "SPAN") {
+        window.katex.render(texText.data, mathElements[i], {
+          displayMode: mathElements[i].classList.contains('display'),
+          throwOnError: false,
+          macros: macros,
+          fleqn: false
+        });
+      }
+    }
+  }
+}
+window.Quarto = {
+  typesetMath
+};
+</script>
 
 </head>
 
@@ -542,8 +571,9 @@ <h2 id="toc-title">Table of contents</h2>
   <li><a href="#pre-existing-datasets" id="toc-pre-existing-datasets" class="nav-link" data-scroll-target="#pre-existing-datasets"><span class="header-section-number">6.4.1</span> Pre-existing datasets</a></li>
   <li><a href="#web-scraping" id="toc-web-scraping" class="nav-link" data-scroll-target="#web-scraping"><span class="header-section-number">6.4.2</span> Web Scraping</a></li>
   <li><a href="#crowdsourcing" id="toc-crowdsourcing" class="nav-link" data-scroll-target="#crowdsourcing"><span class="header-section-number">6.4.3</span> Crowdsourcing</a></li>
-  <li><a href="#synthetic-data" id="toc-synthetic-data" class="nav-link" data-scroll-target="#synthetic-data"><span class="header-section-number">6.4.4</span> Synthetic Data</a></li>
-  <li><a href="#case-study-kws" id="toc-case-study-kws" class="nav-link" data-scroll-target="#case-study-kws"><span class="header-section-number">6.4.5</span> Case Study: KWS</a></li>
+  <li><a href="#data-anonymization" id="toc-data-anonymization" class="nav-link" data-scroll-target="#data-anonymization"><span class="header-section-number">6.4.4</span> Data Anonymization</a></li>
+  <li><a href="#synthetic-data" id="toc-synthetic-data" class="nav-link" data-scroll-target="#synthetic-data"><span class="header-section-number">6.4.5</span> Synthetic Data</a></li>
+  <li><a href="#case-study-kws" id="toc-case-study-kws" class="nav-link" data-scroll-target="#case-study-kws"><span class="header-section-number">6.4.6</span> Case Study: KWS</a></li>
   </ul></li>
   <li><a href="#data-ingestion" id="toc-data-ingestion" class="nav-link" data-scroll-target="#data-ingestion"><span class="header-section-number">6.5</span> Data Ingestion</a>
   <ul>
@@ -628,7 +658,7 @@ <h1 class="title"><span id="sec-data_engineering" class="quarto-section-identifi
 <section id="purpose" class="level2 unnumbered">
 <h2 class="unnumbered anchored" data-anchor-id="purpose">Purpose</h2>
 <p><em>How does data shape ML systems engineering?</em></p>
-<p>In the field of machine learning, data engineering is often overshadowed by the allure of sophisticated algorithms, when in fact data plays a foundational role in determining an AI system’s capabilities and limitations. We need to undersatnd the core principles of data in ML systems, exploring how the acquisition, processing, storage, and governance of data directly impact the performance, reliability, and ethical considerations of AI systems. By understanding these fundamental concepts, we can unlock the true potential of AI and build a solid foundation of high-quality ML solutions.</p>
+<p>In the field of machine learning, data engineering is often overshadowed by the allure of sophisticated algorithms, when in fact data plays a foundational role in determining an AI system’s capabilities and limitations. We need to understand the core principles of data in ML systems, exploring how the acquisition, processing, storage, and governance of data directly impact the performance, reliability, and ethical considerations of AI systems. By understanding these fundamental concepts, we can unlock the true potential of AI and build a solid foundation of high-quality ML solutions.</p>
 <div class="callout callout-style-default callout-tip callout-titled">
 <div class="callout-header d-flex align-content-center">
 <div class="callout-icon-container">
@@ -768,23 +798,6 @@ <h2 data-number="6.2" class="anchored" data-anchor-id="problem-definition"><span
 Nayak, Prateeth, Takuya Higuchi, Anmol Gupta, Shivesh Ranjan, Stephen Shum, Siddharth Sigtia, Erik Marchi, et al. 2022. <span>“Improving Voice Trigger Detection with Metric Learning.”</span> <em>arXiv Preprint arXiv:2204.02455</em>, April. <a href="http://arxiv.org/abs/2204.02455v2">http://arxiv.org/abs/2204.02455v2</a>.
 </div></div><p>The KWS example illustrates the broader principles of problem definition, showing how initial decisions about data requirements ripple throughout a project’s lifecycle. By carefully considering each aspect—from core problem identification through performance benchmarks to deployment constraints—teams can build a strong foundation for their ML systems. The methodical problem definition process provides a framework applicable across the ML spectrum. Whether developing computer vision systems for medical diagnostics, recommendation engines processing millions of user interactions, or natural language models analyzing diverse text corpora, this structured approach helps teams anticipate and plan for their data needs.</p>
 <p>This brings us to data pipelines—the foundational infrastructure that transforms raw data into ML—ready formats while maintaining quality and reliability throughout the process. These pipelines implement our carefully defined requirements in production systems, handling everything from initial data ingestion to final feature generation.</p>
-<div id="exr-kws" class="callout callout-style-default callout-caution callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-2-contents" aria-controls="callout-2" aria-expanded="false" aria-label="Toggle callout">
-<div class="callout-icon-container">
-<i class="callout-icon"></i>
-</div>
-<div class="callout-title-container flex-fill">
-Exercise&nbsp;6.1: Keyword Spotting with TensorFlow Lite Micro
-</div>
-<div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
-</div>
-<div id="callout-2" class="callout-2-contents callout-collapse collapse">
-<div class="callout-body-container callout-body">
-<p>Explore a hands-on guide for building and deploying Keyword Spotting systems using TensorFlow Lite Micro. Follow steps from data collection to model training and deployment to microcontrollers. Learn to create efficient KWS models that recognize specific keywords amidst background noise. Perfect for those interested in machine learning on embedded systems. Unlock the potential of voice-enabled devices with TensorFlow Lite Micro! </p>
-<p><a href="https://colab.research.google.com/drive/17I7GL8WTieGzXYKRtQM2FrFi3eLQIrOM"><img src="https://colab.research.google.com/assets/colab-badge.png" class="img-fluid"></a></p>
-</div>
-</div>
-</div>
 </section>
 <section id="pipeline-fundamentals" class="level2" data-number="6.3">
 <h2 data-number="6.3" class="anchored" data-anchor-id="pipeline-fundamentals"><span class="header-section-number">6.3</span> Pipeline Fundamentals</h2>
@@ -853,16 +866,16 @@ <h3 data-number="6.4.2" class="anchored" data-anchor-id="web-scraping"><span cla
 </figure>
 </div>
 <div id="exr-ws" class="callout callout-style-default callout-caution callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-3-contents" aria-controls="callout-3" aria-expanded="false" aria-label="Toggle callout">
+<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-2-contents" aria-controls="callout-2" aria-expanded="false" aria-label="Toggle callout">
 <div class="callout-icon-container">
 <i class="callout-icon"></i>
 </div>
 <div class="callout-title-container flex-fill">
-Exercise&nbsp;6.2: Web Scraping
+Exercise&nbsp;6.1: Web Scraping
 </div>
 <div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
 </div>
-<div id="callout-3" class="callout-3-contents callout-collapse collapse">
+<div id="callout-2" class="callout-2-contents callout-collapse collapse">
 <div class="callout-body-container callout-body">
 <p>Discover the power of web scraping with Python using libraries like Beautiful Soup and Pandas. This exercise will scrape Python documentation for function names and descriptions and explore NBA player stats. By the end, you’ll have the skills to extract and analyze data from real-world websites. Ready to dive in? Access the Google Colab notebook below and start practicing!<br>
 </p>
@@ -888,8 +901,18 @@ <h3 data-number="6.4.3" class="anchored" data-anchor-id="crowdsourcing"><span cl
 <p>Moreover, when dealing with specialized applications like mobile ML, edge ML, or cloud ML, additional challenges may arise. These applications often require data collected from specific environments or devices, which can be difficult to gather through general crowdsourcing platforms. For example, data for mobile applications utilizing smartphone sensors may necessitate participants with specific hardware features or software versions. Similarly, edge ML systems deployed in industrial settings may require data involving proprietary processes or secure environments, introducing privacy and accessibility challenges.</p>
 <p>Hybrid approaches that combine crowdsourcing with other data collection methods can address these challenges. Organizations may engage specialized communities, partner with relevant stakeholders, or create targeted initiatives to collect domain-specific data. Additionally, synthetic data generation, as discussed in the next section, can augment real-world data when crowdsourcing falls short.</p>
 </section>
-<section id="synthetic-data" class="level3 page-columns page-full" data-number="6.4.4">
-<h3 data-number="6.4.4" class="anchored" data-anchor-id="synthetic-data"><span class="header-section-number">6.4.4</span> Synthetic Data</h3>
+<section id="data-anonymization" class="level3" data-number="6.4.4">
+<h3 data-number="6.4.4" class="anchored" data-anchor-id="data-anonymization"><span class="header-section-number">6.4.4</span> Data Anonymization</h3>
+<p>Protecting the privacy of individuals while still enabling data-driven insights is a central challenge in the modern data landscape. As organizations collect and analyze vast quantities of information, the risk of exposing sensitive details—either accidentally or through malicious attacks—heightens. To mitigate these risks, practitioners have developed a commonly used range of anonymization techniques. These methods transform datasets such that individual identities and sensitive attributes become difficult or nearly impossible to re-identify, while preserving, to varying extents, the overall utility of the data for analysis.</p>
+<p>Masking involves altering or obfuscating sensitive values so that they cannot be directly traced back to the original data subject. For instance, digits in financial account numbers or credit card numbers can be replaced with asterisks, a fixed set of dummy characters, or hashed values to protect sensitive information during display or logging. This anonymization technique is straightforward to implement and understand while clearly protecting identifiable values from being viewed, but may struggle with protecting broader context (e.g.&nbsp;relationships between data points).</p>
+<p>Generalization reduces the precision or granularity of data to decrease the likelihood of re-identification. Instead of revealing an exact date of birth or address, the data is aggregated into broader categories (e.g., age ranges, zip code prefixes). For example, a user’s exact age of 37 might be generalized to an age range of 30-39, while their exact address might be bucketed into a city level granularity. This technique clearly reduces the risk of identifying an individual by sharing data in aggregated form; however, we might consequently lose analytical prediction. Furthermore, if granularity is not chosen correctly, individuals may still be able to be identified under certain conditions.</p>
+<p>Pseudonymization is the process of replacing direct identifiers (like names, Social Security numbers, or email addresses) with artificial identifiers, or “pseudonyms.” These pseudonyms must not reveal, or be easily traceable to, the original data subject. This is commonly used in health records or in any situation where datasets need personal identities removed, but maintain unique entries. This approach allow maintaining individual-level data for analysis (since records can be traced through pseudonyms), while reducing the risk of direct identification. However, if the “key” linking the pseudonym to the real identifier is compromised, re-identification becomes possible.</p>
+<p><span class="math inline">\(k\)</span>-anonymity ensures that each record in a dataset is indistinguishable from at least <span class="math inline">\(𝑘−1\)</span> other records. This is achieved by suppressing or generalizing quasi-identifiers, or attributes that, in combination, could be used to re-identify an individual (e.g., zip code, age, gender). For example, if <span class="math inline">\(k=5\)</span>, every record in the dataset must share the same combination of quasi-identifiers with at least four other records. Thus, an attacker cannot pinpoint a single individual simply by looking at these attributes. This approach provides a formal privacy guaruntee that helps reduce chances of individual re-identification. However, it is extremely high touch and may require a significant level of data distortion and does not protect against things like <a href="https://en.wikipedia.org/wiki/K-anonymity#Attacks">homogeneity or background knowledge attacks</a>.</p>
+<p>Differential privacy (DP) adds carefully <a href="https://digitalprivacy.ieee.org/publications/topics/what-is-differential-privacy#:~:text=At%20its%20roots%2C%20differential%20privacy,a%20result%20of%20providing%20data.">calibrated “noise” or randomized data perturbations</a> to query results or datasets. The goal is to ensure that the inclusion or exclusion of any single individual’s data does not significantly affect the output, thereby concealing their presence. Introduced noise is controled by the <span class="math inline">\(\epsilon\)</span> parameter in <span class="math inline">\(\epsilon\)</span>-Differential Privacy, balancing data utility and privacy guarantees. The clear advantages this approach provides are strong mathematical guarantees of privacy, and DP is widely used in academic and industrial settings (e.g., large-scale data analysis). However, the added noise can affect data accuracy and subsequent model performance; proper parameter tuning is crucial to ensure both privacy and usefulness.</p>
+<p>In summary, effective data anonymization is a balancing act between privacy and utility. Techniques such as masking, generalization, pseudonymization, k-anonymity, and differential privacy each target different aspects of re-identification risk. By carefully selecting and combining these methods, organizations can responsibly derive value from sensitive datasets while respecting the privacy rights and expectations of the individuals represented within them.</p>
+</section>
+<section id="synthetic-data" class="level3 page-columns page-full" data-number="6.4.5">
+<h3 data-number="6.4.5" class="anchored" data-anchor-id="synthetic-data"><span class="header-section-number">6.4.5</span> Synthetic Data</h3>
 <p>Synthetic data generation has emerged as a powerful tool for addressing limitations in data collection, particularly in machine learning applications where real-world data is scarce, expensive, or ethically challenging to obtain. This approach involves creating artificial data using algorithms, simulations, or generative models to mimic real-world datasets. The generated data can be used to supplement or replace real-world data, expanding the possibilities for training robust and accurate machine learning systems. <a href="#fig-synthetic-data" class="quarto-xref">Figure&nbsp;<span>6.7</span></a> illustrates the process of combining synthetic data with historical datasets to create larger, more diverse training sets.</p>
 <div id="fig-synthetic-data" class="quarto-float quarto-figure quarto-figure-center anchored">
 <figure class="quarto-float quarto-float-fig figure">
@@ -909,27 +932,9 @@ <h3 data-number="6.4.4" class="anchored" data-anchor-id="synthetic-data"><span c
 </div><div id="fn4"><p><sup>4</sup>&nbsp;<strong>General Data Protection Regulation (GDPR):</strong> A regulation in EU law on data protection and privacy in the European Union and the European Economic Area.</p></div><div id="fn5"><p><sup>5</sup>&nbsp;<strong>Health Insurance Portability and Accountability Act (HIPAA):</strong> A US law designed to provide privacy standards to protect patients’ medical records and other health information.</p></div></div><p>In addition to expanding datasets, synthetic data addresses critical ethical and privacy concerns. Unlike real-world data, synthetic data attempts to not tie back to specific individuals or entities. This makes it especially useful in sensitive domains such as finance, healthcare, or human resources, where data confidentiality is paramount. The ability to preserve statistical properties while removing identifying information allows researchers to maintain high ethical standards without compromising the quality of their models. In healthcare, privacy regulations such as <a href="https://gdpr.eu/">GDPR</a><a href="#fn4" class="footnote-ref" id="fnref4" role="doc-noteref"><sup>4</sup></a> and <a href="https://www.hhs.gov/hipaa/index.html">HIPAA</a><a href="#fn5" class="footnote-ref" id="fnref5" role="doc-noteref"><sup>5</sup></a> limit the sharing of sensitive patient information. Synthetic data generation enables the creation of realistic yet anonymized datasets that can be used for training diagnostic models without compromising patient privacy.</p>
 <p>Poorly generated data can misrepresent underlying real-world distributions, introducing biases or inaccuracies that degrade model performance. Validating synthetic data against real-world benchmarks is essential to ensure its reliability. Additionally, models trained primarily on synthetic data must be rigorously tested in real-world scenarios to confirm their ability to generalize effectively. Another challenge is the potential amplification of biases present in the original datasets used to inform synthetic data generation. If these biases are not carefully addressed, they may be inadvertently reinforced in the resulting models.</p>
 <p>Synthetic data has revolutionized the way machine learning systems are trained, providing flexibility, diversity, and scalability in data preparation. However, as its adoption grows, practitioners must remain vigilant about its limitations and ethical implications. By combining synthetic data with rigorous validation and thoughtful application, machine learning researchers and engineers can unlock its full potential while ensuring reliability and fairness in their systems.</p>
-<div id="exr-sd" class="callout callout-style-default callout-caution callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-4-contents" aria-controls="callout-4" aria-expanded="false" aria-label="Toggle callout">
-<div class="callout-icon-container">
-<i class="callout-icon"></i>
-</div>
-<div class="callout-title-container flex-fill">
-Exercise&nbsp;6.3: Synthetic Data
-</div>
-<div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
-</div>
-<div id="callout-4" class="callout-4-contents callout-collapse collapse">
-<div class="callout-body-container callout-body">
-<p>Let us learn about synthetic data generation using GANs on tabular data. We’ll take a hands-on approach, diving into the workings of the CTGAN model and applying it to the Synthea dataset from the healthcare domain. From data preprocessing to model training and evaluation, we’ll go step-by-step, learning how to create synthetic data, assess its quality, and unlock the potential of GANs for data augmentation and real-world applications.<br>
-</p>
-<p><a href="https://colab.research.google.com/drive/1nwbvkg32sOUC69zATCfXOygFUBeo0dsx?usp=sharing#scrollTo=TkwYknr44eFn"><img src="https://colab.research.google.com/assets/colab-badge.png" class="img-fluid"></a></p>
-</div>
-</div>
-</div>
 </section>
-<section id="case-study-kws" class="level3 page-columns page-full" data-number="6.4.5">
-<h3 data-number="6.4.5" class="anchored" data-anchor-id="case-study-kws"><span class="header-section-number">6.4.5</span> Case Study: KWS</h3>
+<section id="case-study-kws" class="level3 page-columns page-full" data-number="6.4.6">
+<h3 data-number="6.4.6" class="anchored" data-anchor-id="case-study-kws"><span class="header-section-number">6.4.6</span> Case Study: KWS</h3>
 <p>KWS is an excellent case study of how different data collection approaches can be combined effectively. Each method we’ve discussed plays a role in building robust wake word detection systems, albeit with different trade-offs:</p>
 <p>Pre-existing datasets like Google’s Speech Commands <span class="citation" data-cites="warden2018speech">(<a href="../references.html#ref-warden2018speech" role="doc-biblioref">Warden 2018</a>)</span> provide a foundation for initial development, offering carefully curated voice samples for common wake words. However, these datasets often lack diversity in accents, environments, and languages, necessitating additional data collection strategies.</p>
 <div class="no-row-height column-margin column-container"><div id="ref-warden2018speech" class="csl-entry" role="listitem">
@@ -1421,7 +1426,7 @@ <h2 data-number="6.10" class="anchored" data-anchor-id="conclusion"><span class=
 <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-resource"><span class="header-section-number">6.11</span> Resources</h2>
 <p>Here is a curated list of resources to support students and instructors in their learning and teaching journeys. We are continuously working on expanding this collection and will add new exercises soon.</p>
 <div class="callout callout-style-default callout-note callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-5-contents" aria-controls="callout-5" aria-expanded="true" aria-label="Toggle callout">
+<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-3-contents" aria-controls="callout-3" aria-expanded="true" aria-label="Toggle callout">
 <div class="callout-icon-container">
 <i class="callout-icon"></i>
 </div>
@@ -1430,7 +1435,7 @@ <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-res
 </div>
 <div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
 </div>
-<div id="callout-5" class="callout-5-contents callout-collapse collapse show">
+<div id="callout-3" class="callout-3-contents callout-collapse collapse show">
 <div class="callout-body-container callout-body">
 <p>These slides are a valuable tool for instructors to deliver lectures and for students to review the material at their own pace. We encourage students and instructors to leverage these slides to improve their understanding and facilitate effective knowledge transfer.</p>
 <ul>
@@ -1452,7 +1457,7 @@ <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-res
 </div>
 </div>
 <div class="callout callout-style-default callout-important callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-6-contents" aria-controls="callout-6" aria-expanded="true" aria-label="Toggle callout">
+<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-4-contents" aria-controls="callout-4" aria-expanded="true" aria-label="Toggle callout">
 <div class="callout-icon-container">
 <i class="callout-icon"></i>
 </div>
@@ -1461,7 +1466,7 @@ <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-res
 </div>
 <div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
 </div>
-<div id="callout-6" class="callout-6-contents callout-collapse collapse show">
+<div id="callout-4" class="callout-4-contents callout-collapse collapse show">
 <div class="callout-body-container callout-body">
 <ul>
 <li><em>Coming soon.</em></li>
@@ -1470,7 +1475,7 @@ <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-res
 </div>
 </div>
 <div class="callout callout-style-default callout-caution callout-titled">
-<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-7-contents" aria-controls="callout-7" aria-expanded="true" aria-label="Toggle callout">
+<div class="callout-header d-flex align-content-center" data-bs-toggle="collapse" data-bs-target=".callout-5-contents" aria-controls="callout-5" aria-expanded="true" aria-label="Toggle callout">
 <div class="callout-icon-container">
 <i class="callout-icon"></i>
 </div>
@@ -1479,13 +1484,11 @@ <h2 data-number="6.11" class="anchored" data-anchor-id="sec-data-engineering-res
 </div>
 <div class="callout-btn-toggle d-inline-block border-0 py-1 ps-1 pe-0 float-end"><i class="callout-toggle"></i></div>
 </div>
-<div id="callout-7" class="callout-7-contents callout-collapse collapse show">
+<div id="callout-5" class="callout-5-contents callout-collapse collapse show">
 <div class="callout-body-container callout-body">
 <p>To reinforce the concepts covered in this chapter, we have curated a set of exercises that challenge students to apply their knowledge and deepen their understanding.</p>
 <ul>
-<li><p><a href="#exr-kws" class="quarto-xref">Exercise&nbsp;<span>6.1</span></a></p></li>
-<li><p><a href="#exr-ws" class="quarto-xref">Exercise&nbsp;<span>6.2</span></a></p></li>
-<li><p><a href="#exr-sd" class="quarto-xref">Exercise&nbsp;<span>6.3</span></a></p></li>
+<li><a href="#exr-ws" class="quarto-xref">Exercise&nbsp;<span>6.1</span></a></li>
 </ul>
 </div>
 </div>
diff --git a/docs/index.html b/docs/index.html
index fb17be3c..981df960 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -7,7 +7,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
 <meta name="author" content="Vijay Janapa Reddi">
-<meta name="dcterms.date" content="2025-01-21">
+<meta name="dcterms.date" content="2025-01-23">
 
 <title>Machine Learning Systems</title>
 <style>
@@ -554,7 +554,7 @@ <h1 class="title">Machine Learning Systems</h1>
     <div>
     <div class="quarto-title-meta-heading">Last Updated</div>
     <div class="quarto-title-meta-contents">
-      <p class="date">January 21, 2025</p>
+      <p class="date">January 23, 2025</p>
     </div>
   </div>
   
diff --git a/docs/search.json b/docs/search.json
index cad06239..4d26c71c 100644
--- a/docs/search.json
+++ b/docs/search.json
@@ -844,7 +844,7 @@
     "href": "contents/core/data_engineering/data_engineering.html",
     "title": "6  Data Engineering",
     "section": "",
-    "text": "Purpose\nResources: Slides, Videos, Exercises\nHow does data shape ML systems engineering?\nIn the field of machine learning, data engineering is often overshadowed by the allure of sophisticated algorithms, when in fact data plays a foundational role in determining an AI system’s capabilities and limitations. We need to undersatnd the core principles of data in ML systems, exploring how the acquisition, processing, storage, and governance of data directly impact the performance, reliability, and ethical considerations of AI systems. By understanding these fundamental concepts, we can unlock the true potential of AI and build a solid foundation of high-quality ML solutions.",
+    "text": "Purpose\nResources: Slides, Videos, Exercises\nHow does data shape ML systems engineering?\nIn the field of machine learning, data engineering is often overshadowed by the allure of sophisticated algorithms, when in fact data plays a foundational role in determining an AI system’s capabilities and limitations. We need to understand the core principles of data in ML systems, exploring how the acquisition, processing, storage, and governance of data directly impact the performance, reliability, and ethical considerations of AI systems. By understanding these fundamental concepts, we can unlock the true potential of AI and build a solid foundation of high-quality ML solutions.",
     "crumbs": [
       "<span class='chapter-number'>6</span>  <span class='chapter-title'>Data Engineering</span>"
     ]
@@ -874,7 +874,7 @@
     "href": "contents/core/data_engineering/data_engineering.html#problem-definition",
     "title": "6  Data Engineering",
     "section": "6.2 Problem Definition",
-    "text": "6.2 Problem Definition\nAs discussed in the overview, Sambasivan et al. (2021) observes that neglecting the fundamental importance of data quality gives rise to “Data Cascades” — events where lapses in data quality compound, leading to negative downstream consequences such as flawed predictions, project terminations, and even potential harm to communities. Despite many ML professionals recognizing the importance of data, numerous practitioners report facing these cascades.\nFigure 6.2 illustrates these potential data pitfalls at every stage and how they influence the entire process down the line. The influence of data collection errors is especially pronounced. As illustrated in the figure, any lapses in this initial stage will become apparent at later stages (in model evaluation and deployment) and might lead to costly consequences, such as abandoning the entire model and restarting anew. Therefore, investing in data engineering techniques from the onset will help us detect errors early, mitigating these cascading effects.\n\n\n\n\n\n\nFigure 6.2: Data cascades: compounded costs. Source: Sambasivan et al. (2021).\n\n\nSambasivan, Nithya, Shivani Kapania, Hannah Highfill, Diana Akrong, Praveen Paritosh, and Lora M Aroyo. 2021. “‘Everyone Wants to Do the Model Work, Not the Data Work’: Data Cascades in High-Stakes AI.” In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems, 1–15. ACM. https://doi.org/10.1145/3411764.3445518.\n\n\nThis emphasis on data quality and proper problem definition is fundamental across all types of ML systems. As Sculley et al. (2021) emphasize, it is important to distinguish ML-specific problem framing from the broader context of general software development. Whether developing recommendation engines processing millions of user interactions, computer vision systems analyzing medical images, or natural language models handling diverse text data, each system brings unique challenges that must be carefully considered from the outset. Production ML systems are particularly sensitive to data quality issues, as they must handle continuous data streams, maintain consistent processing pipelines, and adapt to evolving patterns while maintaining performance standards.\n\nSculley, David, Gary Holt, Daniel Golovin, Eugene Davydov, Todd Phillips, Dietmar Ebner, Vinay Chaudhary, Michael Young, Jean-Francois Crespo, and Dan Dennison. 2021. “Technical Debt in Machine Learning Systems.” In Technical Debt in Practice, 177–92. The MIT Press. https://doi.org/10.7551/mitpress/12440.003.0011.\nA solid project foundation is essential for setting the trajectory and ensuring the eventual success of any initiative. At the heart of this foundation lies the crucial first step: identifying a clear problem to solve. This could involve challenges like developing a recommendation system that effectively handles cold-start scenarios, or creating a classification model that maintains consistent accuracy across diverse population segments.\nAs we will explore later in this chapter, establishing clear objectives provides a unified direction that guides the entire project. These objectives might include creating representative datasets that account for various real-world scenarios. Equally important is defining specific benchmarks, such as prediction accuracy and system latency, which offer measurable outcomes to gauge progress and success.\nThroughout this process, engaging with stakeholders—from end-users to business leaders—provides invaluable insights that ensure the project remains aligned with real-world needs and expectations.\nIn particular, a cardinal sin in ML is to begin collecting data (or augmenting an existing dataset) without clearly specifying the underlying problem definition to guide the data collection. We identify the key steps that should precede any data collection effort here:\n\nIdentify and clearly state the problem definition\nSet clear objectives to meet\nEstablish success benchmarks\nUnderstand end-user engagement/use\nUnderstand the constraints and limitations of deployment\nPerform data collection.\nIterate and refine.\n\nKeyword Spotting Example\nKeyword Spotting (KWS) is an excellent example to illustrate all of the general steps in action. This technology is critical for voice-enabled interfaces on endpoint devices such as smartphones. Typically functioning as lightweight wake-word engines, KWS systems are constantly active, listening for a specific phrase to trigger further actions.\nAs shown in Figure 6.3, when we say “OK, Google” or “Alexa,” this initiates a process on a microcontroller embedded within the device.\n\n\n\n\n\n\nFigure 6.3: Keyword Spotting example: interacting with Alexa. Source: Amazon.\n\n\n\nBuilding a reliable KWS model is a complex task. It demands a deep understanding of the deployment scenario, encompassing where and how these devices will operate. For instance, a KWS model’s effectiveness is not just about recognizing a word; it’s about discerning it among various accents and background noises, whether in a bustling cafe or amid the blaring sound of a television in a living room or a kitchen where these devices are commonly found. It’s about ensuring that a whispered “Alexa” in the dead of night or a shouted “OK Google” in a noisy marketplace are recognized with equal precision.\nMoreover, many current KWS voice assistants support a limited number of languages, leaving a substantial portion of the world’s linguistic diversity unrepresented. This limitation is partly due to the difficulty in gathering and monetizing data for languages spoken by smaller populations. In the long-tail distribution of languages, most languages have limited or zero speech training data available, making the development of voice assistants challenging.\nKeyword spotting models can run on low-power, low-price microcontrollers, so theoretically voice interfaces could be expanded to a huge gamut of devices worldwide, beyond smartphones and home assistants. But the level of accuracy and robustness that end-users expect hinges on the availability and quality of speech data, and the ability to label the data correctly. Developing a keyword-spotting model for an arbitrary word or phrase in an arbitrary language begins with clearly understanding the problem statement or definition. Using KWS as an example, we can break down each of the steps as follows:\n\nIdentifying the Problem: KWS detects specific keywords amidst ambient sounds and other spoken words. The primary problem is to design a system that can recognize these keywords with high accuracy, low latency, and minimal false positives or negatives, especially when deployed on devices with limited computational resources. A well-specified problem definition for developing a new KWS model should identify the desired keywords along with the envisioned application and deployment scenario.\nSetting Clear Objectives: The objectives for a KWS system might include:\n\nAchieving a specific accuracy rate (e.g., 98% accuracy in keyword detection).\nEnsuring low latency (e.g., keyword detection and response within 200 milliseconds).\nMinimizing power consumption to extend battery life on embedded devices.\nEnsuring the model’s size is optimized for the available memory on the device.\n\nBenchmarks for Success: Establish clear metrics to measure the success of the KWS system. This could include:\n\nTrue Positive Rate: The percentage of correctly identified keywords relative to all spoken keywords.\nFalse Positive Rate: The percentage of non-keywords (including silence, background noise, and out-of-vocabulary words) incorrectly identified as keywords.\nDetection/Error Tradeoff These curves evaluate KWS on streaming audio representative of a real-world deployment scenario, by comparing the number of false accepts per hour (the number of false positives over the total duration of the evaluation audio) against the false rejection rate (the number of missed keywords relative to the number of spoken keywords in the evaluation audio). Nayak et al. (2022) provides one example of this.\nResponse Time: The time taken from keyword utterance to system response.\nPower Consumption: Average power used during keyword detection.\n\nStakeholder Engagement and Understanding: Engage with stakeholders, which include device manufacturers, hardware and software developers, and end-users. Understand their needs, capabilities, and constraints. For instance:\n\nDevice manufacturers might prioritize low power consumption.\nSoftware developers might emphasize ease of integration.\nEnd-users would prioritize accuracy and responsiveness.\n\nUnderstanding the Constraints and Limitations of Embedded Systems: Embedded devices come with their own set of challenges:\n\nMemory Limitations: KWS models must be lightweight to fit within the memory constraints of embedded devices. Typically, KWS models need to be as small as 16KB to fit in the always-on island of the SoC. Moreover, this is just the model size. Additional application code for preprocessing may also need to fit within the memory constraints.\nProcessing Power: The computational capabilities of embedded devices are limited (a few hundred MHz of clock speed), so the KWS model must be optimized for efficiency.\nPower Consumption: Since many embedded devices are battery-powered, the KWS system must be power-efficient.\nEnvironmental Challenges: Devices might be deployed in various environments, from quiet bedrooms to noisy industrial settings. The KWS system must be robust enough to function effectively across these scenarios.\n\nData Collection and Analysis: For a KWS system, the quality and diversity of data are paramount. Considerations might include:\n\nDemographics: Collect data from speakers with various accents across age and gender to ensure wide-ranging recognition support.\nKeyword Variations: People might pronounce keywords differently or express slight variations in the wake word itself. Ensure the dataset captures these nuances.\nBackground Noises: Include or augment data samples with different ambient noises to train the model for real-world scenarios.\n\nIterative Feedback and Refinement: Once a prototype KWS system is developed, it is important to do the following to ensure that the system remains aligned with the defined problem and objectives as the deployment scenarios change over time and as use-cases evolve.\n\nTest it in real-world scenarios\nGather feedback - are some users or deployment scenarios encountering underperformance relative to others?\nIteratively refine the dataset and model\n\n\n\nNayak, Prateeth, Takuya Higuchi, Anmol Gupta, Shivesh Ranjan, Stephen Shum, Siddharth Sigtia, Erik Marchi, et al. 2022. “Improving Voice Trigger Detection with Metric Learning.” arXiv Preprint arXiv:2204.02455, April. http://arxiv.org/abs/2204.02455v2.\nThe KWS example illustrates the broader principles of problem definition, showing how initial decisions about data requirements ripple throughout a project’s lifecycle. By carefully considering each aspect—from core problem identification through performance benchmarks to deployment constraints—teams can build a strong foundation for their ML systems. The methodical problem definition process provides a framework applicable across the ML spectrum. Whether developing computer vision systems for medical diagnostics, recommendation engines processing millions of user interactions, or natural language models analyzing diverse text corpora, this structured approach helps teams anticipate and plan for their data needs.\nThis brings us to data pipelines—the foundational infrastructure that transforms raw data into ML—ready formats while maintaining quality and reliability throughout the process. These pipelines implement our carefully defined requirements in production systems, handling everything from initial data ingestion to final feature generation.\n\n\n\n\n\n\nExercise 6.1: Keyword Spotting with TensorFlow Lite Micro\n\n\n\n\n\nExplore a hands-on guide for building and deploying Keyword Spotting systems using TensorFlow Lite Micro. Follow steps from data collection to model training and deployment to microcontrollers. Learn to create efficient KWS models that recognize specific keywords amidst background noise. Perfect for those interested in machine learning on embedded systems. Unlock the potential of voice-enabled devices with TensorFlow Lite Micro!",
+    "text": "6.2 Problem Definition\nAs discussed in the overview, Sambasivan et al. (2021) observes that neglecting the fundamental importance of data quality gives rise to “Data Cascades” — events where lapses in data quality compound, leading to negative downstream consequences such as flawed predictions, project terminations, and even potential harm to communities. Despite many ML professionals recognizing the importance of data, numerous practitioners report facing these cascades.\nFigure 6.2 illustrates these potential data pitfalls at every stage and how they influence the entire process down the line. The influence of data collection errors is especially pronounced. As illustrated in the figure, any lapses in this initial stage will become apparent at later stages (in model evaluation and deployment) and might lead to costly consequences, such as abandoning the entire model and restarting anew. Therefore, investing in data engineering techniques from the onset will help us detect errors early, mitigating these cascading effects.\n\n\n\n\n\n\nFigure 6.2: Data cascades: compounded costs. Source: Sambasivan et al. (2021).\n\n\nSambasivan, Nithya, Shivani Kapania, Hannah Highfill, Diana Akrong, Praveen Paritosh, and Lora M Aroyo. 2021. “‘Everyone Wants to Do the Model Work, Not the Data Work’: Data Cascades in High-Stakes AI.” In Proceedings of the 2021 CHI Conference on Human Factors in Computing Systems, 1–15. ACM. https://doi.org/10.1145/3411764.3445518.\n\n\nThis emphasis on data quality and proper problem definition is fundamental across all types of ML systems. As Sculley et al. (2021) emphasize, it is important to distinguish ML-specific problem framing from the broader context of general software development. Whether developing recommendation engines processing millions of user interactions, computer vision systems analyzing medical images, or natural language models handling diverse text data, each system brings unique challenges that must be carefully considered from the outset. Production ML systems are particularly sensitive to data quality issues, as they must handle continuous data streams, maintain consistent processing pipelines, and adapt to evolving patterns while maintaining performance standards.\n\nSculley, David, Gary Holt, Daniel Golovin, Eugene Davydov, Todd Phillips, Dietmar Ebner, Vinay Chaudhary, Michael Young, Jean-Francois Crespo, and Dan Dennison. 2021. “Technical Debt in Machine Learning Systems.” In Technical Debt in Practice, 177–92. The MIT Press. https://doi.org/10.7551/mitpress/12440.003.0011.\nA solid project foundation is essential for setting the trajectory and ensuring the eventual success of any initiative. At the heart of this foundation lies the crucial first step: identifying a clear problem to solve. This could involve challenges like developing a recommendation system that effectively handles cold-start scenarios, or creating a classification model that maintains consistent accuracy across diverse population segments.\nAs we will explore later in this chapter, establishing clear objectives provides a unified direction that guides the entire project. These objectives might include creating representative datasets that account for various real-world scenarios. Equally important is defining specific benchmarks, such as prediction accuracy and system latency, which offer measurable outcomes to gauge progress and success.\nThroughout this process, engaging with stakeholders—from end-users to business leaders—provides invaluable insights that ensure the project remains aligned with real-world needs and expectations.\nIn particular, a cardinal sin in ML is to begin collecting data (or augmenting an existing dataset) without clearly specifying the underlying problem definition to guide the data collection. We identify the key steps that should precede any data collection effort here:\n\nIdentify and clearly state the problem definition\nSet clear objectives to meet\nEstablish success benchmarks\nUnderstand end-user engagement/use\nUnderstand the constraints and limitations of deployment\nPerform data collection.\nIterate and refine.\n\nKeyword Spotting Example\nKeyword Spotting (KWS) is an excellent example to illustrate all of the general steps in action. This technology is critical for voice-enabled interfaces on endpoint devices such as smartphones. Typically functioning as lightweight wake-word engines, KWS systems are constantly active, listening for a specific phrase to trigger further actions.\nAs shown in Figure 6.3, when we say “OK, Google” or “Alexa,” this initiates a process on a microcontroller embedded within the device.\n\n\n\n\n\n\nFigure 6.3: Keyword Spotting example: interacting with Alexa. Source: Amazon.\n\n\n\nBuilding a reliable KWS model is a complex task. It demands a deep understanding of the deployment scenario, encompassing where and how these devices will operate. For instance, a KWS model’s effectiveness is not just about recognizing a word; it’s about discerning it among various accents and background noises, whether in a bustling cafe or amid the blaring sound of a television in a living room or a kitchen where these devices are commonly found. It’s about ensuring that a whispered “Alexa” in the dead of night or a shouted “OK Google” in a noisy marketplace are recognized with equal precision.\nMoreover, many current KWS voice assistants support a limited number of languages, leaving a substantial portion of the world’s linguistic diversity unrepresented. This limitation is partly due to the difficulty in gathering and monetizing data for languages spoken by smaller populations. In the long-tail distribution of languages, most languages have limited or zero speech training data available, making the development of voice assistants challenging.\nKeyword spotting models can run on low-power, low-price microcontrollers, so theoretically voice interfaces could be expanded to a huge gamut of devices worldwide, beyond smartphones and home assistants. But the level of accuracy and robustness that end-users expect hinges on the availability and quality of speech data, and the ability to label the data correctly. Developing a keyword-spotting model for an arbitrary word or phrase in an arbitrary language begins with clearly understanding the problem statement or definition. Using KWS as an example, we can break down each of the steps as follows:\n\nIdentifying the Problem: KWS detects specific keywords amidst ambient sounds and other spoken words. The primary problem is to design a system that can recognize these keywords with high accuracy, low latency, and minimal false positives or negatives, especially when deployed on devices with limited computational resources. A well-specified problem definition for developing a new KWS model should identify the desired keywords along with the envisioned application and deployment scenario.\nSetting Clear Objectives: The objectives for a KWS system might include:\n\nAchieving a specific accuracy rate (e.g., 98% accuracy in keyword detection).\nEnsuring low latency (e.g., keyword detection and response within 200 milliseconds).\nMinimizing power consumption to extend battery life on embedded devices.\nEnsuring the model’s size is optimized for the available memory on the device.\n\nBenchmarks for Success: Establish clear metrics to measure the success of the KWS system. This could include:\n\nTrue Positive Rate: The percentage of correctly identified keywords relative to all spoken keywords.\nFalse Positive Rate: The percentage of non-keywords (including silence, background noise, and out-of-vocabulary words) incorrectly identified as keywords.\nDetection/Error Tradeoff These curves evaluate KWS on streaming audio representative of a real-world deployment scenario, by comparing the number of false accepts per hour (the number of false positives over the total duration of the evaluation audio) against the false rejection rate (the number of missed keywords relative to the number of spoken keywords in the evaluation audio). Nayak et al. (2022) provides one example of this.\nResponse Time: The time taken from keyword utterance to system response.\nPower Consumption: Average power used during keyword detection.\n\nStakeholder Engagement and Understanding: Engage with stakeholders, which include device manufacturers, hardware and software developers, and end-users. Understand their needs, capabilities, and constraints. For instance:\n\nDevice manufacturers might prioritize low power consumption.\nSoftware developers might emphasize ease of integration.\nEnd-users would prioritize accuracy and responsiveness.\n\nUnderstanding the Constraints and Limitations of Embedded Systems: Embedded devices come with their own set of challenges:\n\nMemory Limitations: KWS models must be lightweight to fit within the memory constraints of embedded devices. Typically, KWS models need to be as small as 16KB to fit in the always-on island of the SoC. Moreover, this is just the model size. Additional application code for preprocessing may also need to fit within the memory constraints.\nProcessing Power: The computational capabilities of embedded devices are limited (a few hundred MHz of clock speed), so the KWS model must be optimized for efficiency.\nPower Consumption: Since many embedded devices are battery-powered, the KWS system must be power-efficient.\nEnvironmental Challenges: Devices might be deployed in various environments, from quiet bedrooms to noisy industrial settings. The KWS system must be robust enough to function effectively across these scenarios.\n\nData Collection and Analysis: For a KWS system, the quality and diversity of data are paramount. Considerations might include:\n\nDemographics: Collect data from speakers with various accents across age and gender to ensure wide-ranging recognition support.\nKeyword Variations: People might pronounce keywords differently or express slight variations in the wake word itself. Ensure the dataset captures these nuances.\nBackground Noises: Include or augment data samples with different ambient noises to train the model for real-world scenarios.\n\nIterative Feedback and Refinement: Once a prototype KWS system is developed, it is important to do the following to ensure that the system remains aligned with the defined problem and objectives as the deployment scenarios change over time and as use-cases evolve.\n\nTest it in real-world scenarios\nGather feedback - are some users or deployment scenarios encountering underperformance relative to others?\nIteratively refine the dataset and model\n\n\n\nNayak, Prateeth, Takuya Higuchi, Anmol Gupta, Shivesh Ranjan, Stephen Shum, Siddharth Sigtia, Erik Marchi, et al. 2022. “Improving Voice Trigger Detection with Metric Learning.” arXiv Preprint arXiv:2204.02455, April. http://arxiv.org/abs/2204.02455v2.\nThe KWS example illustrates the broader principles of problem definition, showing how initial decisions about data requirements ripple throughout a project’s lifecycle. By carefully considering each aspect—from core problem identification through performance benchmarks to deployment constraints—teams can build a strong foundation for their ML systems. The methodical problem definition process provides a framework applicable across the ML spectrum. Whether developing computer vision systems for medical diagnostics, recommendation engines processing millions of user interactions, or natural language models analyzing diverse text corpora, this structured approach helps teams anticipate and plan for their data needs.\nThis brings us to data pipelines—the foundational infrastructure that transforms raw data into ML—ready formats while maintaining quality and reliability throughout the process. These pipelines implement our carefully defined requirements in production systems, handling everything from initial data ingestion to final feature generation.",
     "crumbs": [
       "<span class='chapter-number'>6</span>  <span class='chapter-title'>Data Engineering</span>"
     ]
@@ -894,7 +894,7 @@
     "href": "contents/core/data_engineering/data_engineering.html#data-sources",
     "title": "6  Data Engineering",
     "section": "6.4 Data Sources",
-    "text": "6.4 Data Sources\nThe first stage of the pipeline architecture sourcing appropriate data to meet the training needs. The quality and diversity of this data will fundamentally determine our ML system’s learning and prediction capabilities and limitations. ML systems can obtain their training data through several different approaches, each with their own advantages and challenges. Let’s examine each of these approaches in detail.\n\n6.4.1 Pre-existing datasets\nPlatforms like Kaggle and UCI Machine Learning Repository provide ML practitioners with ready-to-use datasets that can jumpstart system development. These pre-existing datasets are particularly valuable when building ML systems as they offer immediate access to cleaned, formatted data with established benchmarks. One of their primary advantages is cost efficiency – creating datasets from scratch requires significant time and resources, especially when building production ML systems that need large amounts of high-quality training data.\nMany of these datasets, such as ImageNet, have become standard benchmarks in the machine learning community, enabling consistent performance comparisons across different models and architectures. For ML system developers, this standardization provides clear metrics for evaluating model improvements and system performance. The immediate availability of these datasets allows teams to begin experimentation and prototyping without delays in data collection and preprocessing.\nHowever, ML practitioners must carefully consider the quality assurance aspects of pre-existing datasets. For instance, the ImageNet dataset was found to have label errors on 6.4% of the validation set (Northcutt, Athalye, and Mueller 2021). While popular datasets benefit from community scrutiny that helps identify and correct errors and biases, most datasets remain “untended gardens” where quality issues can significantly impact downstream system performance if not properly addressed. Moreover, as (Gebru et al. 2021) highlighted in her paper, simply providing a dataset without documentation can lead to misuse and misinterpretation, potentially amplifying biases present in the data.\n\nGebru, Timnit, Jamie Morgenstern, Briana Vecchione, Jennifer Wortman Vaughan, Hanna Wallach, Hal Daumé III, and Kate Crawford. 2021. “Datasheets for Datasets.” Communications of the ACM 64 (12): 86–92. https://doi.org/10.1145/3458723.\n\nPineau, Joelle, Philippe Vincent-Lamarre, Koustuv Sinha, Vincent Larivière, Alina Beygelzimer, Florence d’Alché-Buc, Emily Fox, and Hugo Larochelle. 2021. “Improving Reproducibility in Machine Learning Research (a Report from the Neurips 2019 Reproducibility Program).” Journal of Machine Learning Research 22 (164): 1–20.\nSupporting documentation accompanying existing datasets is invaluable, yet is often only present in widely-used datasets. Good documentation provides insights into the data collection process and variable definitions and sometimes even offers baseline model performances. This information not only aids understanding but also promotes reproducibility in research, a cornerstone of scientific integrity; currently, there is a crisis around improving reproducibility in machine learning systems (Pineau et al. 2021). When other researchers have access to the same data, they can validate findings, test new hypotheses, or apply different methodologies, thus allowing us to build on each other’s work more rapidly.\nWhile existing datasets are invaluable resources, it’s essential to understand the context in which the data was collected. Researchers should be wary of potential overfitting when using popular datasets such as ImageNet (Beyer et al. 2020), leading to inflated performance metrics. Sometimes, these datasets do not reflect the real-world data.\n\nBeyer, Lucas, Olivier J. Hénaff, Alexander Kolesnikov, Xiaohua Zhai, and Aäron van den Oord. 2020. “Are We Done with ImageNet?” arXiv Preprint arXiv:2006.07159, June. http://arxiv.org/abs/2006.07159v1.\nA key consideration for ML systems is how well pre-existing datasets reflect real-world deployment conditions. Relying on standard datasets can create a concerning disconnect between training and production environments. This misalignment becomes particularly problematic when multiple ML systems are trained on the same datasets (Figure 6.5), potentially propagating biases and limitations throughout an entire ecosystem of deployed models.\n\n\n\n\n\n\nFigure 6.5: Training different models on the same dataset.\n\n\n\n\n\n6.4.2 Web Scraping\nWhen building ML systems, particularly in domains where pre-existing datasets are insufficient, web scraping offers a powerful approach to gathering training data at scale. This automated technique for extracting data from websites has become a powerful tool in modern ML system development. It enables teams to build custom datasets tailored to their specific needs.\nWeb scraping has proven particularly valuable for building large-scale ML systems when human-labeled data is scarce. Consider computer vision systems: major datasets like ImageNet and OpenImages were built through systematic web scraping, fundamentally advancing the field of computer vision. In production environments, companies regularly scrape e-commerce sites to gather product images for recognition systems or social media platforms for computer vision applications. Stanford’s LabelMe project demonstrated this approach’s potential early on, scraping Flickr to create a diverse dataset of over 63,000 annotated images.\nThe impact of web scraping extends well beyond computer vision systems. In natural language processing, web-scraped data has enabled the development of increasingly sophisticated ML systems. Large language models, such as ChatGPT and Claude, rely on vast amounts of text scraped from the public internet and media to learn language patterns and generate responses (Groeneveld et al. 2024). Similarly, specialized ML systems like GitHub’s Copilot demonstrate how targeted web scraping—in this case, of code repositories—can create powerful domain-specific assistants (Chen et al. 2021).\n\nGroeneveld, Dirk, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, et al. 2024. “OLMo: Accelerating the Science of Language Models.” arXiv Preprint arXiv:2402.00838, February. http://arxiv.org/abs/2402.00838v4.\n\nChen, Mark, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, et al. 2021. “Evaluating Large Language Models Trained on Code.” arXiv Preprint arXiv:2107.03374, July. http://arxiv.org/abs/2107.03374v2.\nProduction ML systems often require continuous data collection to maintain relevance and performance. Web scraping facilitates this by gathering structured data like stock prices, weather patterns, or product information for analytical applications. However, this continuous collection introduces unique challenges for ML systems. Data consistency becomes crucial—variations in website structure or content formatting can disrupt the data pipeline and affect model performance. Proper data management through databases or warehouses becomes essential not just for storage, but for maintaining data quality and enabling model updates.\nDespite its utility, web scraping presents several challenges that ML system developers must carefully consider. Legal and ethical constraints can limit data collection—not all websites permit scraping, and violating these restrictions can have serious consequences. When building ML systems with scraped data, teams must carefully document data sources and ensure compliance with terms of service and copyright laws. Privacy considerations become particularly critical when dealing with user-generated content, often requiring robust anonymization procedures.\nTechnical limitations also affect the reliability of web-scraped training data. Rate limiting by websites can slow data collection, while the dynamic nature of web content can introduce inconsistencies that impact model training. As shown in Figure 6.6, web scraping can yield unexpected or irrelevant data—such as historical images appearing in contemporary image searches—that can pollute training datasets and degrade model performance. These issues highlight the importance of thorough data validation and cleaning processes in ML pipelines built on web-scraped data.\n\n\n\n\n\n\nFigure 6.6: A picture of old traffic lights (1914). Source: Vox.\n\n\n\n\n\n\n\n\n\nExercise 6.2: Web Scraping\n\n\n\n\n\nDiscover the power of web scraping with Python using libraries like Beautiful Soup and Pandas. This exercise will scrape Python documentation for function names and descriptions and explore NBA player stats. By the end, you’ll have the skills to extract and analyze data from real-world websites. Ready to dive in? Access the Google Colab notebook below and start practicing!\n\n\n\n\n\n\n\n6.4.3 Crowdsourcing\nCrowdsourcing is a collaborative approach to data collection, leveraging the collective efforts of distributed individuals via the internet to tackle tasks requiring human judgment. By engaging a global pool of contributors, this method accelerates the creation of high-quality, labeled datasets for machine learning systems, especially in scenarios where pre-existing data is scarce or domain-specific. Platforms like Amazon Mechanical Turk exemplify how crowdsourcing facilitates this process by distributing annotation tasks to a global workforce. This enables the rapid collection of labels for complex tasks such as sentiment analysis, image recognition, and speech transcription, significantly expediting the data preparation phase.\nOne of the most impactful examples of crowdsourcing in machine learning is the creation of the ImageNet dataset. ImageNet, which revolutionized computer vision, was built by distributing image labeling tasks to contributors via Amazon Mechanical Turk. The contributors categorized millions of images into thousands of classes, enabling researchers to train and benchmark models for a wide variety of visual recognition tasks.\nThe dataset’s availability spurred advancements in deep learning, including the breakthrough AlexNet model in 2012, which demonstrated how large-scale, crowdsourced datasets could drive innovation. ImageNet’s success highlights how leveraging a diverse group of contributors for annotation can enable machine learning systems to achieve unprecedented performance.\nAnother example of crowdsourcing’s potential is Google’s Crowdsource, a platform where volunteers contribute labeled data to improve AI systems in applications like language translation, handwriting recognition, and image understanding. By gamifying the process and engaging global participants, Google harnesses diverse datasets, particularly for underrepresented languages. This approach not only enhances the quality of AI systems but also empowers communities by enabling their contributions to influence technological development.\nCrowdsourcing has also been instrumental in applications beyond traditional dataset annotation. For instance, the navigation app Waze uses crowdsourced data from its users to provide real-time traffic updates, route suggestions, and incident reporting. While this involves dynamic data collection rather than static dataset labeling, it demonstrates how crowdsourcing can generate continuously updated datasets essential for applications like mobile or edge ML systems. These systems often require real-time input to maintain relevance and accuracy in changing environments.\nOne of the primary advantages of crowdsourcing is its scalability. By distributing microtasks to a large audience, projects can process enormous volumes of data quickly and cost-effectively. This scalability is particularly beneficial for machine learning systems that require extensive datasets to achieve high performance. Additionally, the diversity of contributors introduces a wide range of perspectives, cultural insights, and linguistic variations, enriching datasets and improving models’ ability to generalize across populations.\nFlexibility is a key benefit of crowdsourcing. Tasks can be adjusted dynamically based on initial results, allowing for iterative improvements in data collection. For example, Google’s reCAPTCHA system uses crowdsourcing to verify human users while simultaneously labeling datasets for training machine learning models. Users identify objects in images—such as street signs or cars—contributing to the training of autonomous systems. This clever integration demonstrates how crowdsourcing can scale seamlessly when embedded into everyday workflows.\nDespite its advantages, crowdsourcing presents challenges that require careful management. Quality control is a major concern, as the variability in contributors’ expertise and attention can lead to inconsistent or inaccurate annotations. Providing clear instructions and training materials helps ensure participants understand the task requirements. Techniques such as embedding known test cases, leveraging consensus algorithms, or using redundant annotations can mitigate quality issues and align the process with the problem definition discussed earlier.\nEthical considerations are paramount in crowdsourcing, especially when datasets are built at scale using global contributors. It is essential to ensure that participants are fairly compensated for their work and that they are informed about how their contributions will be used. Additionally, privacy concerns must be addressed, particularly when dealing with sensitive or personal information. Transparent sourcing practices, clear communication with contributors, and robust auditing mechanisms are crucial for building trust and maintaining ethical standards.\nThe issue of fair compensation and ethical data sourcing was brought into sharp focus during the development of large-scale AI systems like OpenAI’s ChatGPT. Reports revealed that OpenAI outsourced data annotation tasks to workers in Kenya, employing them to moderate content and identify harmful or inappropriate material that the model might generate. This involved reviewing and labeling distressing content, such as graphic violence and explicit material, to train the AI in recognizing and avoiding such outputs. While this approach enabled OpenAI to improve the safety and utility of ChatGPT, significant ethical concerns arose around the working conditions, the nature of the tasks, and the compensation provided to Kenyan workers.\nMany of the contributors were reportedly paid as little as $1.32 per hour for reviewing and labeling highly traumatic material. The emotional toll of such work, coupled with low wages, raised serious questions about the fairness and transparency of the crowdsourcing process. This controversy highlights a critical gap in ethical crowdsourcing practices. The workers, often from economically disadvantaged regions, were not adequately supported to cope with the psychological impact of their tasks. The lack of mental health resources and insufficient compensation underscored the power imbalances that can emerge when outsourcing data annotation tasks to lower-income regions.\nThe challenges highlighted by the ChatGPT—Kenya controversy are not unique to OpenAI. Many organizations that rely on crowdsourcing for data annotation face similar issues. As machine learning systems grow more complex and require larger datasets, the demand for annotated data will continue to increase. This shows the need for industry-wide standards and best practices to ensure ethical data sourcing. This case emphasizes the importance of considering the human labor behind AI systems. While crowdsourcing offers scalability and diversity, it also brings ethical responsibilities that cannot be overlooked. Organizations must prioritize the well-being and fair treatment of contributors as they build the datasets that drive AI innovation.\nMoreover, when dealing with specialized applications like mobile ML, edge ML, or cloud ML, additional challenges may arise. These applications often require data collected from specific environments or devices, which can be difficult to gather through general crowdsourcing platforms. For example, data for mobile applications utilizing smartphone sensors may necessitate participants with specific hardware features or software versions. Similarly, edge ML systems deployed in industrial settings may require data involving proprietary processes or secure environments, introducing privacy and accessibility challenges.\nHybrid approaches that combine crowdsourcing with other data collection methods can address these challenges. Organizations may engage specialized communities, partner with relevant stakeholders, or create targeted initiatives to collect domain-specific data. Additionally, synthetic data generation, as discussed in the next section, can augment real-world data when crowdsourcing falls short.\n\n\n6.4.4 Synthetic Data\nSynthetic data generation has emerged as a powerful tool for addressing limitations in data collection, particularly in machine learning applications where real-world data is scarce, expensive, or ethically challenging to obtain. This approach involves creating artificial data using algorithms, simulations, or generative models to mimic real-world datasets. The generated data can be used to supplement or replace real-world data, expanding the possibilities for training robust and accurate machine learning systems. Figure 6.7 illustrates the process of combining synthetic data with historical datasets to create larger, more diverse training sets.\n\n\n\n\n\n\nFigure 6.7: Increasing training data size with synthetic data generation. Source: AnyLogic.\n\n\n\nAdvancements in generative modeling techniques, such as diffusion models and flow-matching algorithms1, Generative Adversarial Networks (GANs)2, and Variational Autoencoders (VAEs)3, have greatly enhanced the quality of synthetic data. These techniques can produce data that closely resembles real-world distributions, making it suitable for applications ranging from computer vision to natural language processing. For example, GANs have been used to generate synthetic images for object recognition tasks, creating diverse datasets that are almost indistinguishable from real-world images. Similarly, synthetic data has been leveraged to simulate speech patterns, enhancing the robustness of voice recognition systems.\n1 Diffusion models use noise prediction across time to simulate generation, while flow-matching algorithms minimize the displacement between source and target distributions.2 Generative Adversarial Networks (GANs): Machine learning models with a generator creating data and a discriminator assessing its realism.3 Variational Autoencoders (VAEs): Generative models that encode data into a latent space and decode it to generate new samples.Synthetic data has become particularly valuable in domains where obtaining real-world data is either impractical or costly. The automotive industry has embraced synthetic data to train autonomous vehicle systems; there are only so many cars you can physically crash to get crash-test data that might help an ML system know how to avoid crashes in the first place. Capturing real-world scenarios, especially rare edge cases such as near-accidents or unusual road conditions, is inherently difficult. Synthetic data allows researchers to simulate these scenarios in a controlled virtual environment, ensuring that models are trained to handle a wide range of conditions. This approach has proven invaluable for advancing the capabilities of self-driving cars.\nAnother important application of synthetic data lies in augmenting existing datasets. Introducing variations into datasets enhances model robustness by exposing the model to diverse conditions. For instance, in speech recognition, data augmentation techniques like SpecAugment (Park et al. 2019) introduce noise, shifts, or pitch variations, enabling models to generalize better across different environments and speaker styles. This principle extends to other domains as well, where synthetic data can fill gaps in underrepresented scenarios or edge cases.\n\nPark, Daniel S., William Chan, Yu Zhang, Chung-Cheng Chiu, Barret Zoph, Ekin D. Cubuk, and Quoc V. Le. 2019. “SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition.” arXiv Preprint arXiv:1904.08779, April. http://arxiv.org/abs/1904.08779v3.\n4 General Data Protection Regulation (GDPR): A regulation in EU law on data protection and privacy in the European Union and the European Economic Area.5 Health Insurance Portability and Accountability Act (HIPAA): A US law designed to provide privacy standards to protect patients’ medical records and other health information.In addition to expanding datasets, synthetic data addresses critical ethical and privacy concerns. Unlike real-world data, synthetic data attempts to not tie back to specific individuals or entities. This makes it especially useful in sensitive domains such as finance, healthcare, or human resources, where data confidentiality is paramount. The ability to preserve statistical properties while removing identifying information allows researchers to maintain high ethical standards without compromising the quality of their models. In healthcare, privacy regulations such as GDPR4 and HIPAA5 limit the sharing of sensitive patient information. Synthetic data generation enables the creation of realistic yet anonymized datasets that can be used for training diagnostic models without compromising patient privacy.\nPoorly generated data can misrepresent underlying real-world distributions, introducing biases or inaccuracies that degrade model performance. Validating synthetic data against real-world benchmarks is essential to ensure its reliability. Additionally, models trained primarily on synthetic data must be rigorously tested in real-world scenarios to confirm their ability to generalize effectively. Another challenge is the potential amplification of biases present in the original datasets used to inform synthetic data generation. If these biases are not carefully addressed, they may be inadvertently reinforced in the resulting models.\nSynthetic data has revolutionized the way machine learning systems are trained, providing flexibility, diversity, and scalability in data preparation. However, as its adoption grows, practitioners must remain vigilant about its limitations and ethical implications. By combining synthetic data with rigorous validation and thoughtful application, machine learning researchers and engineers can unlock its full potential while ensuring reliability and fairness in their systems.\n\n\n\n\n\n\nExercise 6.3: Synthetic Data\n\n\n\n\n\nLet us learn about synthetic data generation using GANs on tabular data. We’ll take a hands-on approach, diving into the workings of the CTGAN model and applying it to the Synthea dataset from the healthcare domain. From data preprocessing to model training and evaluation, we’ll go step-by-step, learning how to create synthetic data, assess its quality, and unlock the potential of GANs for data augmentation and real-world applications.\n\n\n\n\n\n\n\n6.4.5 Case Study: KWS\nKWS is an excellent case study of how different data collection approaches can be combined effectively. Each method we’ve discussed plays a role in building robust wake word detection systems, albeit with different trade-offs:\nPre-existing datasets like Google’s Speech Commands (Warden 2018) provide a foundation for initial development, offering carefully curated voice samples for common wake words. However, these datasets often lack diversity in accents, environments, and languages, necessitating additional data collection strategies.\n\nWarden, Pete. 2018. “Speech Commands: A Dataset for Limited-Vocabulary Speech Recognition.” arXiv Preprint arXiv:1804.03209, April. http://arxiv.org/abs/1804.03209v1.\nWeb scraping can supplement these baseline datasets by gathering diverse voice samples from video platforms, podcast repositories, and speech databases. This helps capture natural speech patterns and wake word variations, though careful attention must be paid to audio quality and privacy considerations when scraping voice data.\nCrowdsourcing becomes valuable for collecting specific wake word samples across different demographics and environments. Platforms like Amazon Mechanical Turk can engage contributors to record wake words in various accents, speaking styles, and background conditions. This approach is particularly useful for gathering data for underrepresented languages or specific acoustic environments.\nSynthetic data generation helps fill remaining gaps by creating unlimited variations of wake word utterances. Using speech synthesis (Werchniak et al. 2021) and audio augmentation techniques, developers can generate training data that captures different acoustic environments (busy streets, quiet rooms, moving vehicles), speaker characteristics (age, accent, gender), and background noise conditions.\n\nWerchniak, Andrew, Roberto Barra Chicote, Yuriy Mishchenko, Jasha Droppo, Jeff Condal, Peng Liu, and Anish Shah. 2021. “Exploring the Application of Synthetic Audio in Training Keyword Spotters.” In ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), 7993–96. IEEE; IEEE. https://doi.org/10.1109/icassp39728.2021.9413448.\nThis multi-faceted approach to data collection enables the development of KWS systems that perform robustly across diverse real-world conditions. The combination of methods helps address the unique challenges of wake word detection, from handling various accents and background noise to maintaining consistent performance across different devices and environments.",
+    "text": "6.4 Data Sources\nThe first stage of the pipeline architecture sourcing appropriate data to meet the training needs. The quality and diversity of this data will fundamentally determine our ML system’s learning and prediction capabilities and limitations. ML systems can obtain their training data through several different approaches, each with their own advantages and challenges. Let’s examine each of these approaches in detail.\n\n6.4.1 Pre-existing datasets\nPlatforms like Kaggle and UCI Machine Learning Repository provide ML practitioners with ready-to-use datasets that can jumpstart system development. These pre-existing datasets are particularly valuable when building ML systems as they offer immediate access to cleaned, formatted data with established benchmarks. One of their primary advantages is cost efficiency – creating datasets from scratch requires significant time and resources, especially when building production ML systems that need large amounts of high-quality training data.\nMany of these datasets, such as ImageNet, have become standard benchmarks in the machine learning community, enabling consistent performance comparisons across different models and architectures. For ML system developers, this standardization provides clear metrics for evaluating model improvements and system performance. The immediate availability of these datasets allows teams to begin experimentation and prototyping without delays in data collection and preprocessing.\nHowever, ML practitioners must carefully consider the quality assurance aspects of pre-existing datasets. For instance, the ImageNet dataset was found to have label errors on 6.4% of the validation set (Northcutt, Athalye, and Mueller 2021). While popular datasets benefit from community scrutiny that helps identify and correct errors and biases, most datasets remain “untended gardens” where quality issues can significantly impact downstream system performance if not properly addressed. Moreover, as (Gebru et al. 2021) highlighted in her paper, simply providing a dataset without documentation can lead to misuse and misinterpretation, potentially amplifying biases present in the data.\n\nGebru, Timnit, Jamie Morgenstern, Briana Vecchione, Jennifer Wortman Vaughan, Hanna Wallach, Hal Daumé III, and Kate Crawford. 2021. “Datasheets for Datasets.” Communications of the ACM 64 (12): 86–92. https://doi.org/10.1145/3458723.\n\nPineau, Joelle, Philippe Vincent-Lamarre, Koustuv Sinha, Vincent Larivière, Alina Beygelzimer, Florence d’Alché-Buc, Emily Fox, and Hugo Larochelle. 2021. “Improving Reproducibility in Machine Learning Research (a Report from the Neurips 2019 Reproducibility Program).” Journal of Machine Learning Research 22 (164): 1–20.\nSupporting documentation accompanying existing datasets is invaluable, yet is often only present in widely-used datasets. Good documentation provides insights into the data collection process and variable definitions and sometimes even offers baseline model performances. This information not only aids understanding but also promotes reproducibility in research, a cornerstone of scientific integrity; currently, there is a crisis around improving reproducibility in machine learning systems (Pineau et al. 2021). When other researchers have access to the same data, they can validate findings, test new hypotheses, or apply different methodologies, thus allowing us to build on each other’s work more rapidly.\nWhile existing datasets are invaluable resources, it’s essential to understand the context in which the data was collected. Researchers should be wary of potential overfitting when using popular datasets such as ImageNet (Beyer et al. 2020), leading to inflated performance metrics. Sometimes, these datasets do not reflect the real-world data.\n\nBeyer, Lucas, Olivier J. Hénaff, Alexander Kolesnikov, Xiaohua Zhai, and Aäron van den Oord. 2020. “Are We Done with ImageNet?” arXiv Preprint arXiv:2006.07159, June. http://arxiv.org/abs/2006.07159v1.\nA key consideration for ML systems is how well pre-existing datasets reflect real-world deployment conditions. Relying on standard datasets can create a concerning disconnect between training and production environments. This misalignment becomes particularly problematic when multiple ML systems are trained on the same datasets (Figure 6.5), potentially propagating biases and limitations throughout an entire ecosystem of deployed models.\n\n\n\n\n\n\nFigure 6.5: Training different models on the same dataset.\n\n\n\n\n\n6.4.2 Web Scraping\nWhen building ML systems, particularly in domains where pre-existing datasets are insufficient, web scraping offers a powerful approach to gathering training data at scale. This automated technique for extracting data from websites has become a powerful tool in modern ML system development. It enables teams to build custom datasets tailored to their specific needs.\nWeb scraping has proven particularly valuable for building large-scale ML systems when human-labeled data is scarce. Consider computer vision systems: major datasets like ImageNet and OpenImages were built through systematic web scraping, fundamentally advancing the field of computer vision. In production environments, companies regularly scrape e-commerce sites to gather product images for recognition systems or social media platforms for computer vision applications. Stanford’s LabelMe project demonstrated this approach’s potential early on, scraping Flickr to create a diverse dataset of over 63,000 annotated images.\nThe impact of web scraping extends well beyond computer vision systems. In natural language processing, web-scraped data has enabled the development of increasingly sophisticated ML systems. Large language models, such as ChatGPT and Claude, rely on vast amounts of text scraped from the public internet and media to learn language patterns and generate responses (Groeneveld et al. 2024). Similarly, specialized ML systems like GitHub’s Copilot demonstrate how targeted web scraping—in this case, of code repositories—can create powerful domain-specific assistants (Chen et al. 2021).\n\nGroeneveld, Dirk, Iz Beltagy, Pete Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Harsh Jha, et al. 2024. “OLMo: Accelerating the Science of Language Models.” arXiv Preprint arXiv:2402.00838, February. http://arxiv.org/abs/2402.00838v4.\n\nChen, Mark, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, et al. 2021. “Evaluating Large Language Models Trained on Code.” arXiv Preprint arXiv:2107.03374, July. http://arxiv.org/abs/2107.03374v2.\nProduction ML systems often require continuous data collection to maintain relevance and performance. Web scraping facilitates this by gathering structured data like stock prices, weather patterns, or product information for analytical applications. However, this continuous collection introduces unique challenges for ML systems. Data consistency becomes crucial—variations in website structure or content formatting can disrupt the data pipeline and affect model performance. Proper data management through databases or warehouses becomes essential not just for storage, but for maintaining data quality and enabling model updates.\nDespite its utility, web scraping presents several challenges that ML system developers must carefully consider. Legal and ethical constraints can limit data collection—not all websites permit scraping, and violating these restrictions can have serious consequences. When building ML systems with scraped data, teams must carefully document data sources and ensure compliance with terms of service and copyright laws. Privacy considerations become particularly critical when dealing with user-generated content, often requiring robust anonymization procedures.\nTechnical limitations also affect the reliability of web-scraped training data. Rate limiting by websites can slow data collection, while the dynamic nature of web content can introduce inconsistencies that impact model training. As shown in Figure 6.6, web scraping can yield unexpected or irrelevant data—such as historical images appearing in contemporary image searches—that can pollute training datasets and degrade model performance. These issues highlight the importance of thorough data validation and cleaning processes in ML pipelines built on web-scraped data.\n\n\n\n\n\n\nFigure 6.6: A picture of old traffic lights (1914). Source: Vox.\n\n\n\n\n\n\n\n\n\nExercise 6.1: Web Scraping\n\n\n\n\n\nDiscover the power of web scraping with Python using libraries like Beautiful Soup and Pandas. This exercise will scrape Python documentation for function names and descriptions and explore NBA player stats. By the end, you’ll have the skills to extract and analyze data from real-world websites. Ready to dive in? Access the Google Colab notebook below and start practicing!\n\n\n\n\n\n\n\n6.4.3 Crowdsourcing\nCrowdsourcing is a collaborative approach to data collection, leveraging the collective efforts of distributed individuals via the internet to tackle tasks requiring human judgment. By engaging a global pool of contributors, this method accelerates the creation of high-quality, labeled datasets for machine learning systems, especially in scenarios where pre-existing data is scarce or domain-specific. Platforms like Amazon Mechanical Turk exemplify how crowdsourcing facilitates this process by distributing annotation tasks to a global workforce. This enables the rapid collection of labels for complex tasks such as sentiment analysis, image recognition, and speech transcription, significantly expediting the data preparation phase.\nOne of the most impactful examples of crowdsourcing in machine learning is the creation of the ImageNet dataset. ImageNet, which revolutionized computer vision, was built by distributing image labeling tasks to contributors via Amazon Mechanical Turk. The contributors categorized millions of images into thousands of classes, enabling researchers to train and benchmark models for a wide variety of visual recognition tasks.\nThe dataset’s availability spurred advancements in deep learning, including the breakthrough AlexNet model in 2012, which demonstrated how large-scale, crowdsourced datasets could drive innovation. ImageNet’s success highlights how leveraging a diverse group of contributors for annotation can enable machine learning systems to achieve unprecedented performance.\nAnother example of crowdsourcing’s potential is Google’s Crowdsource, a platform where volunteers contribute labeled data to improve AI systems in applications like language translation, handwriting recognition, and image understanding. By gamifying the process and engaging global participants, Google harnesses diverse datasets, particularly for underrepresented languages. This approach not only enhances the quality of AI systems but also empowers communities by enabling their contributions to influence technological development.\nCrowdsourcing has also been instrumental in applications beyond traditional dataset annotation. For instance, the navigation app Waze uses crowdsourced data from its users to provide real-time traffic updates, route suggestions, and incident reporting. While this involves dynamic data collection rather than static dataset labeling, it demonstrates how crowdsourcing can generate continuously updated datasets essential for applications like mobile or edge ML systems. These systems often require real-time input to maintain relevance and accuracy in changing environments.\nOne of the primary advantages of crowdsourcing is its scalability. By distributing microtasks to a large audience, projects can process enormous volumes of data quickly and cost-effectively. This scalability is particularly beneficial for machine learning systems that require extensive datasets to achieve high performance. Additionally, the diversity of contributors introduces a wide range of perspectives, cultural insights, and linguistic variations, enriching datasets and improving models’ ability to generalize across populations.\nFlexibility is a key benefit of crowdsourcing. Tasks can be adjusted dynamically based on initial results, allowing for iterative improvements in data collection. For example, Google’s reCAPTCHA system uses crowdsourcing to verify human users while simultaneously labeling datasets for training machine learning models. Users identify objects in images—such as street signs or cars—contributing to the training of autonomous systems. This clever integration demonstrates how crowdsourcing can scale seamlessly when embedded into everyday workflows.\nDespite its advantages, crowdsourcing presents challenges that require careful management. Quality control is a major concern, as the variability in contributors’ expertise and attention can lead to inconsistent or inaccurate annotations. Providing clear instructions and training materials helps ensure participants understand the task requirements. Techniques such as embedding known test cases, leveraging consensus algorithms, or using redundant annotations can mitigate quality issues and align the process with the problem definition discussed earlier.\nEthical considerations are paramount in crowdsourcing, especially when datasets are built at scale using global contributors. It is essential to ensure that participants are fairly compensated for their work and that they are informed about how their contributions will be used. Additionally, privacy concerns must be addressed, particularly when dealing with sensitive or personal information. Transparent sourcing practices, clear communication with contributors, and robust auditing mechanisms are crucial for building trust and maintaining ethical standards.\nThe issue of fair compensation and ethical data sourcing was brought into sharp focus during the development of large-scale AI systems like OpenAI’s ChatGPT. Reports revealed that OpenAI outsourced data annotation tasks to workers in Kenya, employing them to moderate content and identify harmful or inappropriate material that the model might generate. This involved reviewing and labeling distressing content, such as graphic violence and explicit material, to train the AI in recognizing and avoiding such outputs. While this approach enabled OpenAI to improve the safety and utility of ChatGPT, significant ethical concerns arose around the working conditions, the nature of the tasks, and the compensation provided to Kenyan workers.\nMany of the contributors were reportedly paid as little as $1.32 per hour for reviewing and labeling highly traumatic material. The emotional toll of such work, coupled with low wages, raised serious questions about the fairness and transparency of the crowdsourcing process. This controversy highlights a critical gap in ethical crowdsourcing practices. The workers, often from economically disadvantaged regions, were not adequately supported to cope with the psychological impact of their tasks. The lack of mental health resources and insufficient compensation underscored the power imbalances that can emerge when outsourcing data annotation tasks to lower-income regions.\nThe challenges highlighted by the ChatGPT—Kenya controversy are not unique to OpenAI. Many organizations that rely on crowdsourcing for data annotation face similar issues. As machine learning systems grow more complex and require larger datasets, the demand for annotated data will continue to increase. This shows the need for industry-wide standards and best practices to ensure ethical data sourcing. This case emphasizes the importance of considering the human labor behind AI systems. While crowdsourcing offers scalability and diversity, it also brings ethical responsibilities that cannot be overlooked. Organizations must prioritize the well-being and fair treatment of contributors as they build the datasets that drive AI innovation.\nMoreover, when dealing with specialized applications like mobile ML, edge ML, or cloud ML, additional challenges may arise. These applications often require data collected from specific environments or devices, which can be difficult to gather through general crowdsourcing platforms. For example, data for mobile applications utilizing smartphone sensors may necessitate participants with specific hardware features or software versions. Similarly, edge ML systems deployed in industrial settings may require data involving proprietary processes or secure environments, introducing privacy and accessibility challenges.\nHybrid approaches that combine crowdsourcing with other data collection methods can address these challenges. Organizations may engage specialized communities, partner with relevant stakeholders, or create targeted initiatives to collect domain-specific data. Additionally, synthetic data generation, as discussed in the next section, can augment real-world data when crowdsourcing falls short.\n\n\n6.4.4 Data Anonymization\nProtecting the privacy of individuals while still enabling data-driven insights is a central challenge in the modern data landscape. As organizations collect and analyze vast quantities of information, the risk of exposing sensitive details—either accidentally or through malicious attacks—heightens. To mitigate these risks, practitioners have developed a commonly used range of anonymization techniques. These methods transform datasets such that individual identities and sensitive attributes become difficult or nearly impossible to re-identify, while preserving, to varying extents, the overall utility of the data for analysis.\nMasking involves altering or obfuscating sensitive values so that they cannot be directly traced back to the original data subject. For instance, digits in financial account numbers or credit card numbers can be replaced with asterisks, a fixed set of dummy characters, or hashed values to protect sensitive information during display or logging. This anonymization technique is straightforward to implement and understand while clearly protecting identifiable values from being viewed, but may struggle with protecting broader context (e.g. relationships between data points).\nGeneralization reduces the precision or granularity of data to decrease the likelihood of re-identification. Instead of revealing an exact date of birth or address, the data is aggregated into broader categories (e.g., age ranges, zip code prefixes). For example, a user’s exact age of 37 might be generalized to an age range of 30-39, while their exact address might be bucketed into a city level granularity. This technique clearly reduces the risk of identifying an individual by sharing data in aggregated form; however, we might consequently lose analytical prediction. Furthermore, if granularity is not chosen correctly, individuals may still be able to be identified under certain conditions.\nPseudonymization is the process of replacing direct identifiers (like names, Social Security numbers, or email addresses) with artificial identifiers, or “pseudonyms.” These pseudonyms must not reveal, or be easily traceable to, the original data subject. This is commonly used in health records or in any situation where datasets need personal identities removed, but maintain unique entries. This approach allow maintaining individual-level data for analysis (since records can be traced through pseudonyms), while reducing the risk of direct identification. However, if the “key” linking the pseudonym to the real identifier is compromised, re-identification becomes possible.\n\\(k\\)-anonymity ensures that each record in a dataset is indistinguishable from at least \\(𝑘−1\\) other records. This is achieved by suppressing or generalizing quasi-identifiers, or attributes that, in combination, could be used to re-identify an individual (e.g., zip code, age, gender). For example, if \\(k=5\\), every record in the dataset must share the same combination of quasi-identifiers with at least four other records. Thus, an attacker cannot pinpoint a single individual simply by looking at these attributes. This approach provides a formal privacy guaruntee that helps reduce chances of individual re-identification. However, it is extremely high touch and may require a significant level of data distortion and does not protect against things like homogeneity or background knowledge attacks.\nDifferential privacy (DP) adds carefully calibrated “noise” or randomized data perturbations to query results or datasets. The goal is to ensure that the inclusion or exclusion of any single individual’s data does not significantly affect the output, thereby concealing their presence. Introduced noise is controled by the \\(\\epsilon\\) parameter in \\(\\epsilon\\)-Differential Privacy, balancing data utility and privacy guarantees. The clear advantages this approach provides are strong mathematical guarantees of privacy, and DP is widely used in academic and industrial settings (e.g., large-scale data analysis). However, the added noise can affect data accuracy and subsequent model performance; proper parameter tuning is crucial to ensure both privacy and usefulness.\nIn summary, effective data anonymization is a balancing act between privacy and utility. Techniques such as masking, generalization, pseudonymization, k-anonymity, and differential privacy each target different aspects of re-identification risk. By carefully selecting and combining these methods, organizations can responsibly derive value from sensitive datasets while respecting the privacy rights and expectations of the individuals represented within them.\n\n\n6.4.5 Synthetic Data\nSynthetic data generation has emerged as a powerful tool for addressing limitations in data collection, particularly in machine learning applications where real-world data is scarce, expensive, or ethically challenging to obtain. This approach involves creating artificial data using algorithms, simulations, or generative models to mimic real-world datasets. The generated data can be used to supplement or replace real-world data, expanding the possibilities for training robust and accurate machine learning systems. Figure 6.7 illustrates the process of combining synthetic data with historical datasets to create larger, more diverse training sets.\n\n\n\n\n\n\nFigure 6.7: Increasing training data size with synthetic data generation. Source: AnyLogic.\n\n\n\nAdvancements in generative modeling techniques, such as diffusion models and flow-matching algorithms1, Generative Adversarial Networks (GANs)2, and Variational Autoencoders (VAEs)3, have greatly enhanced the quality of synthetic data. These techniques can produce data that closely resembles real-world distributions, making it suitable for applications ranging from computer vision to natural language processing. For example, GANs have been used to generate synthetic images for object recognition tasks, creating diverse datasets that are almost indistinguishable from real-world images. Similarly, synthetic data has been leveraged to simulate speech patterns, enhancing the robustness of voice recognition systems.\n1 Diffusion models use noise prediction across time to simulate generation, while flow-matching algorithms minimize the displacement between source and target distributions.2 Generative Adversarial Networks (GANs): Machine learning models with a generator creating data and a discriminator assessing its realism.3 Variational Autoencoders (VAEs): Generative models that encode data into a latent space and decode it to generate new samples.Synthetic data has become particularly valuable in domains where obtaining real-world data is either impractical or costly. The automotive industry has embraced synthetic data to train autonomous vehicle systems; there are only so many cars you can physically crash to get crash-test data that might help an ML system know how to avoid crashes in the first place. Capturing real-world scenarios, especially rare edge cases such as near-accidents or unusual road conditions, is inherently difficult. Synthetic data allows researchers to simulate these scenarios in a controlled virtual environment, ensuring that models are trained to handle a wide range of conditions. This approach has proven invaluable for advancing the capabilities of self-driving cars.\nAnother important application of synthetic data lies in augmenting existing datasets. Introducing variations into datasets enhances model robustness by exposing the model to diverse conditions. For instance, in speech recognition, data augmentation techniques like SpecAugment (Park et al. 2019) introduce noise, shifts, or pitch variations, enabling models to generalize better across different environments and speaker styles. This principle extends to other domains as well, where synthetic data can fill gaps in underrepresented scenarios or edge cases.\n\nPark, Daniel S., William Chan, Yu Zhang, Chung-Cheng Chiu, Barret Zoph, Ekin D. Cubuk, and Quoc V. Le. 2019. “SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition.” arXiv Preprint arXiv:1904.08779, April. http://arxiv.org/abs/1904.08779v3.\n4 General Data Protection Regulation (GDPR): A regulation in EU law on data protection and privacy in the European Union and the European Economic Area.5 Health Insurance Portability and Accountability Act (HIPAA): A US law designed to provide privacy standards to protect patients’ medical records and other health information.In addition to expanding datasets, synthetic data addresses critical ethical and privacy concerns. Unlike real-world data, synthetic data attempts to not tie back to specific individuals or entities. This makes it especially useful in sensitive domains such as finance, healthcare, or human resources, where data confidentiality is paramount. The ability to preserve statistical properties while removing identifying information allows researchers to maintain high ethical standards without compromising the quality of their models. In healthcare, privacy regulations such as GDPR4 and HIPAA5 limit the sharing of sensitive patient information. Synthetic data generation enables the creation of realistic yet anonymized datasets that can be used for training diagnostic models without compromising patient privacy.\nPoorly generated data can misrepresent underlying real-world distributions, introducing biases or inaccuracies that degrade model performance. Validating synthetic data against real-world benchmarks is essential to ensure its reliability. Additionally, models trained primarily on synthetic data must be rigorously tested in real-world scenarios to confirm their ability to generalize effectively. Another challenge is the potential amplification of biases present in the original datasets used to inform synthetic data generation. If these biases are not carefully addressed, they may be inadvertently reinforced in the resulting models.\nSynthetic data has revolutionized the way machine learning systems are trained, providing flexibility, diversity, and scalability in data preparation. However, as its adoption grows, practitioners must remain vigilant about its limitations and ethical implications. By combining synthetic data with rigorous validation and thoughtful application, machine learning researchers and engineers can unlock its full potential while ensuring reliability and fairness in their systems.\n\n\n6.4.6 Case Study: KWS\nKWS is an excellent case study of how different data collection approaches can be combined effectively. Each method we’ve discussed plays a role in building robust wake word detection systems, albeit with different trade-offs:\nPre-existing datasets like Google’s Speech Commands (Warden 2018) provide a foundation for initial development, offering carefully curated voice samples for common wake words. However, these datasets often lack diversity in accents, environments, and languages, necessitating additional data collection strategies.\n\nWarden, Pete. 2018. “Speech Commands: A Dataset for Limited-Vocabulary Speech Recognition.” arXiv Preprint arXiv:1804.03209, April. http://arxiv.org/abs/1804.03209v1.\nWeb scraping can supplement these baseline datasets by gathering diverse voice samples from video platforms, podcast repositories, and speech databases. This helps capture natural speech patterns and wake word variations, though careful attention must be paid to audio quality and privacy considerations when scraping voice data.\nCrowdsourcing becomes valuable for collecting specific wake word samples across different demographics and environments. Platforms like Amazon Mechanical Turk can engage contributors to record wake words in various accents, speaking styles, and background conditions. This approach is particularly useful for gathering data for underrepresented languages or specific acoustic environments.\nSynthetic data generation helps fill remaining gaps by creating unlimited variations of wake word utterances. Using speech synthesis (Werchniak et al. 2021) and audio augmentation techniques, developers can generate training data that captures different acoustic environments (busy streets, quiet rooms, moving vehicles), speaker characteristics (age, accent, gender), and background noise conditions.\n\nWerchniak, Andrew, Roberto Barra Chicote, Yuriy Mishchenko, Jasha Droppo, Jeff Condal, Peng Liu, and Anish Shah. 2021. “Exploring the Application of Synthetic Audio in Training Keyword Spotters.” In ICASSP 2021 - 2021 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP), 7993–96. IEEE; IEEE. https://doi.org/10.1109/icassp39728.2021.9413448.\nThis multi-faceted approach to data collection enables the development of KWS systems that perform robustly across diverse real-world conditions. The combination of methods helps address the unique challenges of wake word detection, from handling various accents and background noise to maintaining consistent performance across different devices and environments.",
     "crumbs": [
       "<span class='chapter-number'>6</span>  <span class='chapter-title'>Data Engineering</span>"
     ]
@@ -964,7 +964,7 @@
     "href": "contents/core/data_engineering/data_engineering.html#sec-data-engineering-resource",
     "title": "6  Data Engineering",
     "section": "6.11 Resources",
-    "text": "6.11 Resources\nHere is a curated list of resources to support students and instructors in their learning and teaching journeys. We are continuously working on expanding this collection and will add new exercises soon.\n\n\n\n\n\n\nSlides\n\n\n\n\n\nThese slides are a valuable tool for instructors to deliver lectures and for students to review the material at their own pace. We encourage students and instructors to leverage these slides to improve their understanding and facilitate effective knowledge transfer.\n\nData Engineering: Overview.\nFeature engineering.\nData Standards: Speech Commands.\nCrowdsourcing Data for the Long Tail.\nReusing and Adapting Existing Datasets.\nResponsible Data Collection.\nData Anomaly Detection:\n\nAnomaly Detection: Overview.\nAnomaly Detection: Challenges.\nAnomaly Detection: Datasets.\nAnomaly Detection: using Autoencoders.\n\n\n\n\n\n\n\n\n\n\n\nVideos\n\n\n\n\n\n\nComing soon.\n\n\n\n\n\n\n\n\n\n\nExercises\n\n\n\n\n\nTo reinforce the concepts covered in this chapter, we have curated a set of exercises that challenge students to apply their knowledge and deepen their understanding.\n\nExercise 6.1\nExercise 6.2\nExercise 6.3",
+    "text": "6.11 Resources\nHere is a curated list of resources to support students and instructors in their learning and teaching journeys. We are continuously working on expanding this collection and will add new exercises soon.\n\n\n\n\n\n\nSlides\n\n\n\n\n\nThese slides are a valuable tool for instructors to deliver lectures and for students to review the material at their own pace. We encourage students and instructors to leverage these slides to improve their understanding and facilitate effective knowledge transfer.\n\nData Engineering: Overview.\nFeature engineering.\nData Standards: Speech Commands.\nCrowdsourcing Data for the Long Tail.\nReusing and Adapting Existing Datasets.\nResponsible Data Collection.\nData Anomaly Detection:\n\nAnomaly Detection: Overview.\nAnomaly Detection: Challenges.\nAnomaly Detection: Datasets.\nAnomaly Detection: using Autoencoders.\n\n\n\n\n\n\n\n\n\n\n\nVideos\n\n\n\n\n\n\nComing soon.\n\n\n\n\n\n\n\n\n\n\nExercises\n\n\n\n\n\nTo reinforce the concepts covered in this chapter, we have curated a set of exercises that challenge students to apply their knowledge and deepen their understanding.\n\nExercise 6.1",
     "crumbs": [
       "<span class='chapter-number'>6</span>  <span class='chapter-title'>Data Engineering</span>"
     ]