chore: 🔐 add secret

.github/workflows/ci.yml

@@ -12,6 +12,8 @@ jobs:
   run:
     name: Run
     runs-on: ubuntu-latest
+    env:
+      IUCNREDLIST_TOKEN: ${{ secrets.IUCNREDLIST_TOKEN }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

datadex/__init__.py

@@ -5,6 +5,7 @@
 from dagster_duckdb_pandas import DuckDBPandasIOManager
 
 from . import assets, jobs
+from .resources import HuggingFaceResource
 
 DBT_PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) + "/../dbt/"
 DATA_DIR = os.path.dirname(os.path.abspath(__file__)) + "/../data/"
@@ -15,6 +16,7 @@
 python_assets = load_assets_from_modules([assets])
 
 resources = {
+    "hf": HuggingFaceResource(),
     "dbt": dbt,
     "io_manager": DuckDBPandasIOManager(
         database=DATA_DIR + "local.duckdb",

datadex/assets.py

@@ -84,3 +84,11 @@ def wikidata_asteroids() -> pd.DataFrame:
     )
 
     return pd.read_csv(io.StringIO(response.content.decode("utf-8")))
+
+
+# @asset
+# def hf_co2_data(co2_global_trend: pd.DataFrame, hf: HuggingFaceResource) -> None:
+#     """
+#     CO2 data from Our World in Data.
+#     """
+#     hf.upload_dataset(co2_global_trend, "co2_global_trend")

datadex/resources.py

@@ -1,8 +1,17 @@
-from dagster import ConfigurableResource
+import huggingface_hub as hf_hub
+from dagster import ConfigurableResource, EnvVar
+from datasets import Dataset
 
 
 class HuggingFaceResource(ConfigurableResource):
-    # token: str = EnvVar("HUGGINGFACE_TOKEN")
+    token: str = EnvVar("HUGGINGFACE_TOKEN")
 
     def login(self):
-        raise NotImplementedError()
+        hf_hub.login(token=self.token)
+
+    def upload_dataset(self, dataset, name):
+        self.login()
+        dataset = Dataset.from_pandas(dataset)
+        r = dataset.push_to_hub("davidgasquez/" + name)
+        print(r)
+        return r