From 70e5d7f7c72c0ebb9cda39b9d9f10542f0aac837 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Mon, 7 Oct 2024 19:34:48 +0000 Subject: [PATCH 01/34] update doc --- docs/building/sources/xarray-kerchunk.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/building/sources/xarray-kerchunk.rst b/docs/building/sources/xarray-kerchunk.rst index 44a9417d..fa09037e 100644 --- a/docs/building/sources/xarray-kerchunk.rst +++ b/docs/building/sources/xarray-kerchunk.rst @@ -2,7 +2,7 @@ xarray-kerchunk ################# -.. literalinclude:: xarray-kerchunk.yaml +.. literalinclude:: yaml/xarray-kerchunk.yaml :language: yaml The code below is inspired by the `kerchunk tutorial`_, and makes use of From 3d88c4347e7cb141157f11efcdc46d9d9760643d Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:18:02 +0200 Subject: [PATCH 02/34] logging.. --- .github/workflows/push-to-private.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 7134db57..1e3b0aa5 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -22,6 +22,7 @@ jobs: # env: # MLX_TOKEN: ${{ secrets.MLX_TOKEN }} run: | + git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone git remote add private https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git git fetch private git push private develop From da8678f992fc31992a4f403468b571b202663f31 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:19:44 +0200 Subject: [PATCH 03/34] logging... --- .github/workflows/push-to-private.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 1e3b0aa5..7c2c1a77 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -18,11 +18,16 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - - name: Push changes to private repository - # env: - # MLX_TOKEN: ${{ secrets.MLX_TOKEN }} + - name: clone run: | git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone + + - name: remote-add + run: | + git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + + - name: Push changes to private repository + run: | git remote add private https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git git fetch private git push private develop From 39b6ebaa5ce9a4a13c2a7d6b0cf287ac21ac5f38 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:21:59 +0200 Subject: [PATCH 04/34] push to private --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 7c2c1a77..11d52191 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -30,4 +30,4 @@ jobs: run: | git remote add private https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git git fetch private - git push private develop + git push -u private develop From 5507f0864c92b2202aa90c2e895e6d8533e6ece7 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:23:30 +0200 Subject: [PATCH 05/34] push to private. --- .github/workflows/push-to-private.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 11d52191..28b0a56f 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -25,6 +25,7 @@ jobs: - name: remote-add run: | git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git fetch new - name: Push changes to private repository run: | From 57b19f554a2fb7ddc49df38a12a59d6da490c641 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:24:52 +0200 Subject: [PATCH 06/34] push to private.. --- .github/workflows/push-to-private.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 28b0a56f..2f31fb07 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,6 +23,10 @@ jobs: git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone - name: remote-add + run: | + git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + + - name: fetch run: | git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git git fetch new From cb474bad3f524d36dfbb3ddca92b858da0be84d1 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:32:06 +0200 Subject: [PATCH 07/34] push to private... --- .github/workflows/push-to-private.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 2f31fb07..a2b1a899 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -18,21 +18,25 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + - name: env + run: | + env | sort + - name: clone run: | git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone - - name: remote-add + - name: set-url run: | - git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - name: fetch run: | - git remote add new https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - git fetch new + git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git fetch - name: Push changes to private repository run: | - git remote add private https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - git fetch private - git push -u private develop + git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git fetch + git push develop From 4e37a91b5f11df7bd5386a32792a1f4240b93d69 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:36:14 +0200 Subject: [PATCH 08/34] push to private (wip) --- .github/workflows/push-to-private.yml | 84 +++++++++++++-------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index a2b1a899..d0415c36 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -1,42 +1,42 @@ -name: Push to another repository - -on: - push: - branches: - - develop - -jobs: - push_changes: - runs-on: ubuntu-latest - - steps: - - name: Checkout source repository - uses: actions/checkout@v3 - - - name: Set up Git configuration - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: env - run: | - env | sort - - - name: clone - run: | - git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone - - - name: set-url - run: | - git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - - - name: fetch - run: | - git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - git fetch - - - name: Push changes to private repository - run: | - git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - git fetch - git push develop +# name: Push to another repository +# +# on: +# push: +# branches: +# - develop +# +# jobs: +# push_changes: +# runs-on: ubuntu-latest +# +# steps: +# - name: Checkout source repository +# uses: actions/checkout@v3 +# +# - name: Set up Git configuration +# run: | +# git config user.name "github-actions[bot]" +# git config user.email "github-actions[bot]@users.noreply.github.com" +# +# - name: env +# run: | +# env | sort +# +# - name: clone +# run: | +# git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone +# +# - name: set-url +# run: | +# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git +# +# - name: fetch +# run: | +# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git +# git fetch +# +# - name: Push changes to private repository +# run: | +# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git +# git fetch +# git push develop From f92dea77fc7582c718115b01b606c3e36d156961 Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Tue, 8 Oct 2024 10:37:18 +0200 Subject: [PATCH 09/34] push to private (wip) --- .github/workflows/push-to-private.yml | 84 +++++++++++++-------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index d0415c36..3594bb71 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -1,42 +1,42 @@ -# name: Push to another repository -# -# on: -# push: -# branches: -# - develop -# -# jobs: -# push_changes: -# runs-on: ubuntu-latest -# -# steps: -# - name: Checkout source repository -# uses: actions/checkout@v3 -# -# - name: Set up Git configuration -# run: | -# git config user.name "github-actions[bot]" -# git config user.email "github-actions[bot]@users.noreply.github.com" -# -# - name: env -# run: | -# env | sort -# -# - name: clone -# run: | -# git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone -# -# - name: set-url -# run: | -# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git -# -# - name: fetch -# run: | -# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git -# git fetch -# -# - name: Push changes to private repository -# run: | -# git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git -# git fetch -# git push develop +name: Push to another repository + +on: + push: + branches: + - develop + +jobs: + push_changes: + runs-on: ubuntu-latest + + steps: + - name: Checkout source repository + uses: actions/checkout@v3 + + - name: Set up Git configuration + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # - name: env + # run: | + # env | sort + + # - name: clone + # run: | + # git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone + + # - name: set-url + # run: | + # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + + # - name: fetch + # run: | + # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + # git fetch + + # - name: Push changes to private repository + # run: | + # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + # git fetch + # git push develop From c326972b73954c55cd515bc2816153e4ec6c5af2 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:41:47 +0100 Subject: [PATCH 10/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 31 ++++++++------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 3594bb71..71f49368 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -12,31 +12,18 @@ jobs: steps: - name: Checkout source repository uses: actions/checkout@v3 + with: + fetch-depth: 0 + fetch-tags: true - name: Set up Git configuration run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - # - name: env - # run: | - # env | sort - - # - name: clone - # run: | - # git clone https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git check-clone - - # - name: set-url - # run: | - # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - - # - name: fetch - # run: | - # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - # git fetch - - # - name: Push changes to private repository - # run: | - # git remote set-url origin https://${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - # git fetch - # git push develop + - name: Push changes to private repository + env: + MLX_TOKEN: ${{ secrets.MLX_TOKEN }} + run: | + git remote add private https://x-access-token:${MLX_TOKEN}@github.com/ecmwf-lab/anemoi-datasets-private.git + git push private develop From 065a5b687214580f66b27d5fdbef7bfd3d9a79a7 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:45:42 +0100 Subject: [PATCH 11/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 71f49368..b6d9db7c 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -5,6 +5,9 @@ on: branches: - develop +permissions: + contents: write # Allows pushing to the repository + jobs: push_changes: runs-on: ubuntu-latest From 8658a0f164bf48502bcd6439978d1605a4be5ecc Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:52:18 +0100 Subject: [PATCH 12/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index b6d9db7c..3c05027e 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,10 +23,8 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - + git remote add-url private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + - name: Push changes to private repository - env: - MLX_TOKEN: ${{ secrets.MLX_TOKEN }} run: | - git remote add private https://x-access-token:${MLX_TOKEN}@github.com/ecmwf-lab/anemoi-datasets-private.git - git push private develop + git --verbose push private develop From 46103d720f749e35ac08e1e7233d618cad1b64ac Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:55:30 +0100 Subject: [PATCH 13/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 3c05027e..2c75ab29 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,7 +23,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote add-url private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git remote set-url --add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - name: Push changes to private repository run: | From 5287760ee09fe13e268339d31b70748af9fcfc4a Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:56:44 +0100 Subject: [PATCH 14/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 2c75ab29..2de77a73 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,7 +23,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote set-url --add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git remote set-url private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - name: Push changes to private repository run: | From e6b1751c6e7867652bdfa945938c9dda9fb8c376 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:58:12 +0100 Subject: [PATCH 15/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 2de77a73..71ea1f46 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,7 +23,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote set-url private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git remote add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git - name: Push changes to private repository run: | From e6d0c19b092fe77832e1b5a371578d613830666e Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:00:02 +0100 Subject: [PATCH 16/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 71ea1f46..6eb9c639 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -27,4 +27,4 @@ jobs: - name: Push changes to private repository run: | - git --verbose push private develop + git push --verbose private develop From 250a33aff5aa5063d37f52d10e41dd30c27a8cfd Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:53:23 +0100 Subject: [PATCH 17/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 6eb9c639..61a59088 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -24,6 +24,10 @@ jobs: git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git remote add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + + - name: env + run: + env | sort - name: Push changes to private repository run: | From 05cab31f5076c4e0ad9a02fa80f6547e25141438 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:24:13 +0100 Subject: [PATCH 18/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 61a59088..0e2a4f47 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,7 +23,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf-lab/anemoi-datasets-private.git + git remote add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git - name: env run: From a5611c67ea8d9f62a0ca2fa2b0ff435fea6b7848 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:00:48 +0100 Subject: [PATCH 19/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 0e2a4f47..61d8b383 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,7 +23,7 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote add private https://x-access-token:${{ secrets.MLX_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git + git remote add private https://x-access-token:${{ secrets.SYNC_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git - name: env run: From 3536f42eaeb98b356757f764a8968b64edefd669 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:06:27 +0100 Subject: [PATCH 20/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 61d8b383..a8506e36 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -31,4 +31,6 @@ jobs: - name: Push changes to private repository run: | - git push --verbose private develop + git remote rm origin + git remote add origin https://x-access-token:${{ secrets.SYNC_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git + git push --verbose From d640913a74e3103b2f173e15a923c6a7eb763fa6 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:07:34 +0100 Subject: [PATCH 21/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index a8506e36..3ef7b964 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -33,4 +33,4 @@ jobs: run: | git remote rm origin git remote add origin https://x-access-token:${{ secrets.SYNC_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git - git push --verbose + git push --set-upstream origin develop From f6e37ab832c2526133b83adb03ef1ef87655be75 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:28:32 +0100 Subject: [PATCH 22/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 3ef7b964..e31edc41 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -23,14 +23,13 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - git remote add private https://x-access-token:${{ secrets.SYNC_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git - - - name: env - run: - env | sort + + - name: Setup SSH key + uses: webfactory/ssh-agent@v0.5.0 + with: + ssh-private-key: ${{ secrets.KEY_TO_PRIVATE }} - name: Push changes to private repository run: | - git remote rm origin - git remote add origin https://x-access-token:${{ secrets.SYNC_TOKEN }}@github.com/ecmwf/anemoi-datasets-private.git - git push --set-upstream origin develop + git remote add private git@github.com:ecmwf/anemoi-datasets-private.git + git push --set-upstream private develop From fb530dffe6585484a87a439c3e012a75624b74a1 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:40:10 +0100 Subject: [PATCH 23/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index e31edc41..218a9f20 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -10,6 +10,7 @@ permissions: jobs: push_changes: + if: ${{ !contains(github.repository, 'private') }} runs-on: ubuntu-latest steps: @@ -31,5 +32,5 @@ jobs: - name: Push changes to private repository run: | - git remote add private git@github.com:ecmwf/anemoi-datasets-private.git + git remote add private git@github.com:ecmwf/${{ github.repository }}-private.git git push --set-upstream private develop From 3820ff45cbd72dd0852b92e24b033239ca01d023 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:42:42 +0100 Subject: [PATCH 24/34] Update push-to-private.yml --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 218a9f20..f10242ad 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -32,5 +32,5 @@ jobs: - name: Push changes to private repository run: | - git remote add private git@github.com:ecmwf/${{ github.repository }}-private.git + git remote add private git@github.com:${{ github.repository }}-private.git git push --set-upstream private develop From ab873536c4e35cd442069e1bf4940511f0cf19da Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:43:56 +0100 Subject: [PATCH 25/34] Update ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9eb40fe..32b8f76e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ on: jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: + if: ${{ !contains(github.repository, 'private') }} name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main @@ -45,6 +46,7 @@ jobs: # Build downstream packages on HPC downstream-ci-hpc: + if: ${{ !contains(github.repository, 'private') }} name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main From 14b233c1e4292389865d57a131ae3d393ccb7bb1 Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:47:23 +0100 Subject: [PATCH 26/34] Update ci.yml --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32b8f76e..d792b522 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,9 @@ on: jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: - if: ${{ !contains(github.repository, 'private') }} + name: downstream-ci - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main with: anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }} @@ -46,9 +46,8 @@ jobs: # Build downstream packages on HPC downstream-ci-hpc: - if: ${{ !contains(github.repository, 'private') }} name: downstream-ci-hpc - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }} From 297c73e97c7b290fe76a0e472acd472f86e1edca Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Tue, 8 Oct 2024 12:51:00 +0100 Subject: [PATCH 27/34] Update python-publish.yml --- .github/workflows/python-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 0f65b5d1..3e542f75 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -9,6 +9,7 @@ on: jobs: quality: + if: ${{ !contains(github.repository, 'private') }} uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2 with: skip-hooks: "no-commit-to-branch" From 29d9cd3f26bb5c7176d4e2b0f9e10222710f5b4e Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 8 Oct 2024 11:55:12 +0000 Subject: [PATCH 28/34] update workflow --- .github/workflows/push-to-private.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index f10242ad..4f753363 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -5,9 +5,6 @@ on: branches: - develop -permissions: - contents: write # Allows pushing to the repository - jobs: push_changes: if: ${{ !contains(github.repository, 'private') }} @@ -24,12 +21,12 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - + - name: Setup SSH key uses: webfactory/ssh-agent@v0.5.0 with: ssh-private-key: ${{ secrets.KEY_TO_PRIVATE }} - + - name: Push changes to private repository run: | git remote add private git@github.com:${{ github.repository }}-private.git From 9ba17481bcc1cf786b9a4f0c1e335934f402d570 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 8 Oct 2024 11:56:07 +0000 Subject: [PATCH 29/34] run qa --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d792b522..c37038f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ on: jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: - + name: downstream-ci if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main From 224e35c2d75efdfdbdd9c3c84bc0a513109b8a10 Mon Sep 17 00:00:00 2001 From: Baudouin Raoult Date: Tue, 8 Oct 2024 12:47:47 +0000 Subject: [PATCH 30/34] tidy --- .github/workflows/push-to-private.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml index 4f753363..4cc53efd 100644 --- a/.github/workflows/push-to-private.yml +++ b/.github/workflows/push-to-private.yml @@ -1,4 +1,4 @@ -name: Push to another repository +name: Push to private repository on: push: From 97dc57a878b12cca5debdaa5170c5a2747106c9d Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Wed, 9 Oct 2024 09:16:13 +0200 Subject: [PATCH 31/34] call provenance less often (#77) * call provenance less often --- CHANGELOG.md | 1 + src/anemoi/datasets/create/persistent.py | 5 ++++- src/anemoi/datasets/create/statistics/__init__.py | 5 ++++- src/anemoi/datasets/create/zarr.py | 6 +++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a874a400..b4beac5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Keep it human-readable, your future self will thank you! - Bug fix when creating dataset from zarr - Bug fix with area selection in cutout operation - add paths-ignore to ci workflow +- call provenance less often ### Removed diff --git a/src/anemoi/datasets/create/persistent.py b/src/anemoi/datasets/create/persistent.py index ac3a0c15..61d715f3 100644 --- a/src/anemoi/datasets/create/persistent.py +++ b/src/anemoi/datasets/create/persistent.py @@ -56,8 +56,11 @@ def items(self): yield pickle.load(f) def add_provenance(self, **kwargs): + path = os.path.join(self.dirname, "provenance.json") + if os.path.exists(path): + return out = dict(provenance=gather_provenance_info(), **kwargs) - with open(os.path.join(self.dirname, "provenance.json"), "w") as f: + with open(path, "w") as f: json.dump(out, f) def add(self, elt, *, key): diff --git a/src/anemoi/datasets/create/statistics/__init__.py b/src/anemoi/datasets/create/statistics/__init__.py index e5fcf460..adb55c72 100644 --- a/src/anemoi/datasets/create/statistics/__init__.py +++ b/src/anemoi/datasets/create/statistics/__init__.py @@ -187,8 +187,11 @@ def __init__(self, dirname, overwrite=False): def add_provenance(self, **kwargs): self.create(exist_ok=True) + path = os.path.join(self.dirname, "provenance.json") + if os.path.exists(path): + return out = dict(provenance=gather_provenance_info(), **kwargs) - with open(os.path.join(self.dirname, "provenance.json"), "w") as f: + with open(path, "w") as f: json.dump(out, f) def create(self, exist_ok): diff --git a/src/anemoi/datasets/create/zarr.py b/src/anemoi/datasets/create/zarr.py index 0adfcef8..5759b972 100644 --- a/src/anemoi/datasets/create/zarr.py +++ b/src/anemoi/datasets/create/zarr.py @@ -168,7 +168,11 @@ def reset(self, lengths): return self.create(lengths, overwrite=True) def add_provenance(self, name): + z = self._open_write() + + if name in z.attrs: + return + from anemoi.utils.provenance import gather_provenance_info - z = self._open_write() z.attrs[name] = gather_provenance_info() From 202b45894edbc6f423d0145b2d93a892e06b30bc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 08:17:15 +0100 Subject: [PATCH 32/34] [pre-commit.ci] pre-commit autoupdate (#75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [pre-commit.ci] pre-commit autoupdate updates: - [github.com/pre-commit/pre-commit-hooks: v4.6.0 → v5.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.6.0...v5.0.0) - [github.com/astral-sh/ruff-pre-commit: v0.6.4 → v0.6.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.4...v0.6.9) - [github.com/tox-dev/pyproject-fmt: 2.2.3 → 2.2.4](https://github.com/tox-dev/pyproject-fmt/compare/2.2.3...2.2.4) * Update .pre-commit-config.yaml Co-authored-by: Jonathan Berthias --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jesper Dramsch Co-authored-by: Jonathan Berthias --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b9f40cd..093cd552 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,12 +5,12 @@ repos: - id: clear-notebooks-output name: clear-notebooks-output files: tools/.*\.ipynb$ - stages: [commit] + stages: [pre-commit] language: python entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace additional_dependencies: [jupyter] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml # Check YAML files for syntax errors only args: [--unsafe, --allow-multiple-documents] @@ -40,7 +40,7 @@ repos: - --force-single-line-imports - --profile black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.9 hooks: - id: ruff # Next line if for documenation cod snippets @@ -66,7 +66,7 @@ repos: - id: docconvert args: ["numpy"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.2.3" + rev: "2.2.4" hooks: - id: pyproject-fmt From da7a9c70a0ab8be17f375eb8cd3bb07d7d9949ae Mon Sep 17 00:00:00 2001 From: b8raoult <53792887+b8raoult@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:20:37 +0100 Subject: [PATCH 33/34] Feature/params metadata (#76) * add `variables_metadata` to metadata --- CHANGELOG.md | 4 ++ src/anemoi/datasets/commands/inspect.py | 4 ++ src/anemoi/datasets/create/__init__.py | 43 ++-------------------- src/anemoi/datasets/create/input/result.py | 38 ++++++++++++++++++- src/anemoi/datasets/data/forwards.py | 5 ++- src/anemoi/datasets/data/join.py | 13 +++++++ src/anemoi/datasets/data/select.py | 11 ++++++ src/anemoi/datasets/data/stores.py | 4 ++ src/anemoi/datasets/data/unchecked.py | 5 +++ 9 files changed, 86 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4beac5b..7f59fea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Keep it human-readable, your future self will thank you! - Update documentation +### Changed + +- Add `variables_metadata` entry in the dataset metadata + ## [0.5.5](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...0.5.5) - 2024-10-04 ### Changed diff --git a/src/anemoi/datasets/commands/inspect.py b/src/anemoi/datasets/commands/inspect.py index 35898973..0e89f7b4 100644 --- a/src/anemoi/datasets/commands/inspect.py +++ b/src/anemoi/datasets/commands/inspect.py @@ -506,6 +506,10 @@ def name_to_index(self): def variables(self): return self.metadata["variables"] + @property + def variables_metadata(self): + return self.metadata.get("variables_metadata", {}) + class Version0_12(Version0_6): def details(self): diff --git a/src/anemoi/datasets/create/__init__.py b/src/anemoi/datasets/create/__init__.py index 68f78a9e..14cf5240 100644 --- a/src/anemoi/datasets/create/__init__.py +++ b/src/anemoi/datasets/create/__init__.py @@ -25,6 +25,7 @@ from anemoi.utils.dates import frequency_to_timedelta from anemoi.utils.humanize import compress_dates from anemoi.utils.humanize import seconds_to_human +from anemoi.utils.sanitise import sanitise from earthkit.data.core.order import build_remapping from anemoi.datasets import MissingDateError @@ -52,7 +53,7 @@ LOG = logging.getLogger(__name__) -VERSION = "0.20" +VERSION = "0.30" def json_tidy(o): @@ -325,43 +326,6 @@ def build_input_(main_config, output_config): return builder -def tidy_recipe(config: object): - """Remove potentially private information in the config""" - config = deepcopy(config) - if isinstance(config, (tuple, list)): - return [tidy_recipe(_) for _ in config] - if isinstance(config, (dict, DotDict)): - for k, v in config.items(): - if k.startswith("_"): - config[k] = "*** REMOVED FOR SECURITY ***" - else: - config[k] = tidy_recipe(v) - if isinstance(config, str): - if config.startswith("_"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("s3://"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("gs://"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("http"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("ftp"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("file"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("ssh"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("scp"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("rsync"): - return "*** REMOVED FOR SECURITY ***" - if config.startswith("/"): - return "*** REMOVED FOR SECURITY ***" - if "@" in config: - return "*** REMOVED FOR SECURITY ***" - return config - - class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixin): dataset_class = NewDataset def __init__(self, path, config, check_name=False, overwrite=False, use_threads=False, statistics_temp_dir=None, progress=None, test=False, cache=None, **kwargs): # fmt: skip @@ -448,7 +412,7 @@ def _run(self): metadata.update(self.main_config.get("add_metadata", {})) metadata["_create_yaml_config"] = self.main_config.get_serialisable_dict() - metadata["recipe"] = tidy_recipe(self.main_config.get_serialisable_dict()) + metadata["recipe"] = sanitise(self.main_config.get_serialisable_dict()) metadata["description"] = self.main_config.description metadata["licence"] = self.main_config["licence"] @@ -467,6 +431,7 @@ def _run(self): metadata["data_request"] = self.minimal_input.data_request metadata["field_shape"] = self.minimal_input.field_shape metadata["proj_string"] = self.minimal_input.proj_string + metadata["variables_metadata"] = self.minimal_input.variables_metadata metadata["start_date"] = dates[0].isoformat() metadata["end_date"] = dates[-1].isoformat() diff --git a/src/anemoi/datasets/create/input/result.py b/src/anemoi/datasets/create/input/result.py index ff3ca17f..7fab9acf 100644 --- a/src/anemoi/datasets/create/input/result.py +++ b/src/anemoi/datasets/create/input/result.py @@ -30,6 +30,31 @@ LOG = logging.getLogger(__name__) +def _fields_metatata(variables, cube): + assert isinstance(variables, tuple), variables + + result = {} + for i, c in enumerate(cube.iterate_cubelets()): + assert c._coords_names[1] == variables[i], (c._coords_names[1], variables[i]) + f = cube[c.coords] + md = f.metadata(namespace="mars") + if not md: + md = f.metadata(namespace="default") + + if md.get("param") == "~": + md["param"] = f.metadata("param") + assert md["param"] not in ("~", "unknown"), (md, f.metadata("param")) + + if md.get("param") == "unknown": + md["param"] = str(f.metadata("paramId", default="unknown")) + # assert md['param'] != 'unknown', (md, f.metadata('param')) + + result[variables[i]] = md + + assert i + 1 == len(variables), (i + 1, len(variables)) + return result + + def _data_request(data): date = None params_levels = defaultdict(set) @@ -312,7 +337,10 @@ def _trace_datasource(self, *args, **kwargs): def build_coords(self): if self._coords_already_built: return - from_data = self.get_cube().user_coords + + cube = self.get_cube() + + from_data = cube.user_coords from_config = self.context.order_by keys_from_config = list(from_config.keys()) @@ -359,11 +387,19 @@ def build_coords(self): self._field_shape = first_field.shape self._proj_string = first_field.proj_string if hasattr(first_field, "proj_string") else None + self._cube = cube + + self._coords_already_built = True + @property def variables(self): self.build_coords() return self._variables + @property + def variables_metadata(self): + return _fields_metatata(self.variables, self._cube) + @property def ensembles(self): self.build_coords() diff --git a/src/anemoi/datasets/data/forwards.py b/src/anemoi/datasets/data/forwards.py index bacad3d5..5ccfd7f0 100644 --- a/src/anemoi/datasets/data/forwards.py +++ b/src/anemoi/datasets/data/forwards.py @@ -63,6 +63,10 @@ def name_to_index(self): def variables(self): return self.forward.variables + @property + def variables_metadata(self): + return self.forward.variables_metadata + @property def statistics(self): return self.forward.statistics @@ -253,7 +257,6 @@ def missing(self): offset = 0 result = set() for d in self.datasets: - print("--->", d.missing, d) result.update(offset + m for m in d.missing) if self.axis == 0: # Advance if axis is time offset += len(d) diff --git a/src/anemoi/datasets/data/join.py b/src/anemoi/datasets/data/join.py index c8e514bd..85452166 100644 --- a/src/anemoi/datasets/data/join.py +++ b/src/anemoi/datasets/data/join.py @@ -111,6 +111,19 @@ def variables(self): return result + @cached_property + def variables_metadata(self): + seen = set() + result = {} + for d in reversed(self.datasets): + for v in reversed(d.variables): + while v in seen: + v = f"({v})" + seen.add(v) + result[v] = d.variables_metadata[v] + + return result + @cached_property def name_to_index(self): return {k: i for i, k in enumerate(self.variables)} diff --git a/src/anemoi/datasets/data/select.py b/src/anemoi/datasets/data/select.py index d949dd58..c131bd84 100644 --- a/src/anemoi/datasets/data/select.py +++ b/src/anemoi/datasets/data/select.py @@ -76,6 +76,10 @@ def shape(self): def variables(self): return [self.dataset.variables[i] for i in self.indices] + @cached_property + def variables_metadata(self): + return {k: v for k, v in self.dataset.variables_metadata.items() if k in self.variables} + @cached_property def name_to_index(self): return {k: i for i, k in enumerate(self.variables)} @@ -108,13 +112,20 @@ def __init__(self, dataset, rename): super().__init__(dataset) for n in rename: assert n in dataset.variables, n + self._variables = [rename.get(v, v) for v in dataset.variables] + self._variables_metadata = {rename.get(k, k): v for k, v in dataset.variables_metadata.items()} + self.rename = rename @property def variables(self): return self._variables + @property + def variables_metadata(self): + return self._variables_metadata + @cached_property def name_to_index(self): return {k: i for i, k in enumerate(self.variables)} diff --git a/src/anemoi/datasets/data/stores.py b/src/anemoi/datasets/data/stores.py index 9ced8281..874ac53d 100644 --- a/src/anemoi/datasets/data/stores.py +++ b/src/anemoi/datasets/data/stores.py @@ -302,6 +302,10 @@ def variables(self): ) ] + @property + def variables_metadata(self): + return self.z.attrs.get("variables_metadata", {}) + def __repr__(self): return self.path diff --git a/src/anemoi/datasets/data/unchecked.py b/src/anemoi/datasets/data/unchecked.py index 3f9663db..a0d9b9ff 100644 --- a/src/anemoi/datasets/data/unchecked.py +++ b/src/anemoi/datasets/data/unchecked.py @@ -91,6 +91,11 @@ def name_to_index(self): def variables(self): raise NotImplementedError() + @property + @check("check_same_variables") + def variables_metadata(self): + raise NotImplementedError() + @property @check("check_same_variables") def statistics(self): From 904e102c35b943b8eb7d4b6c5e97de8c9548560d Mon Sep 17 00:00:00 2001 From: Florian Pinault Date: Wed, 9 Oct 2024 19:35:06 +0200 Subject: [PATCH 34/34] Feature/params metadata (#79) * add `variables_metadata` to metadata * fix bad params * use sanitise * updated tests and updated test data accordingly. this needs to be merged to fix the tests in "develop" --------- Co-authored-by: Baudouin Raoult --- CHANGELOG.md | 4 ++++ tests/create/data_sources.yaml | 28 ---------------------------- tests/create/test_create.py | 15 +++++++-------- 3 files changed, 11 insertions(+), 36 deletions(-) delete mode 100644 tests/create/data_sources.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f59fea3..9cee8700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ Keep it human-readable, your future self will thank you! - Add `variables_metadata` entry in the dataset metadata +### Changed + +- Add `variables_metadata` entry in the dataset metadata + ## [0.5.5](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...0.5.5) - 2024-10-04 ### Changed diff --git a/tests/create/data_sources.yaml b/tests/create/data_sources.yaml deleted file mode 100644 index cf1d7735..00000000 --- a/tests/create/data_sources.yaml +++ /dev/null @@ -1,28 +0,0 @@ -common: - mars_request: &mars_request - expver: "0001" - class: ea - grid: 20./20. - -dates: - start: 2020-12-30 00:00:00 - end: 2021-01-03 12:00:00 - frequency: 12h - -data_sources: - my_predefined_source: - mars: - <<: *mars_request - param: [2t] - levtype: sfc - stream: oper - type: an - -input: - forcings: - template: ${data_sources.my_predefined_source} - param: - - cos_latitude - -statistics: - end: 2021 diff --git a/tests/create/test_create.py b/tests/create/test_create.py index 2112cd78..257ab434 100755 --- a/tests/create/test_create.py +++ b/tests/create/test_create.py @@ -98,23 +98,22 @@ def compare_dot_zattrs(a, b, path, errors): if isinstance(a, dict): a_keys = list(a.keys()) b_keys = list(b.keys()) - for k in set(a_keys) & set(b_keys): - if k in [ - "licence", - "total_number_of_files", - ]: + for k in set(a_keys) | set(b_keys): + if k not in a_keys: + errors.append(f"❌ {path}.{k} : missing key (only in reference)") + continue + if k not in b_keys: + errors.append(f"❌ {path}.{k} : additional key (missing in reference)") continue if k in [ "timestamp", "uuid", "latest_write_timestamp", - "_create_yaml_config", "history", "provenance", "provenance_load", "description", "config_path", - "dataset_status", "total_size", ]: if type(a[k]) is not type(b[k]): @@ -217,8 +216,8 @@ def compare(self): raise AssertionError("Comparison failed") compare_datasets(self.ds_output, self.ds_reference) - compare_statistics(self.ds_output, self.ds_reference) + # do not compare tendencies statistics yet, as we don't know yet if they should stay @pytest.mark.parametrize("name", NAMES)