diff --git a/.github/workflows/devskim-security-linter.yml b/.github/workflows/devskim-security-linter.yml new file mode 100644 index 000000000..e7e33b7ab --- /dev/null +++ b/.github/workflows/devskim-security-linter.yml @@ -0,0 +1,37 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party (Microsoft) and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# For more details about Devskim, visit https://github.com/marketplace/actions/devskim + +name: DevSkim + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '00 4 * * *' + +jobs: + lint: + name: DevSkim + runs-on: ubuntu-20.04 + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Run DevSkim scanner + uses: microsoft/DevSkim-Action@v1 + with: + ignore-globs: "**/.git/**,**/test/**" + + - name: Upload DevSkim scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: devskim-results.sarif diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 84e99b614..3046cdf15 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -52,27 +52,20 @@ jobs: steps: - - name: Deploy to Feathr SQL Registry Azure Web App - id: deploy-to-sql-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-sql-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_SQL_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' - - name: Deploy to Feathr Purview Registry Azure Web App id: deploy-to-purview-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-purview-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_PURVIEW_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_PURVIEW_REGISTRY_WEBHOOK }} - name: Deploy to Feathr RBAC Registry Azure Web App id: deploy-to-rbac-webapp - uses: azure/webapps-deploy@v2 - with: - app-name: 'feathr-rbac-registry' - publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_FEATHR_RBAC_REGISTRY }} - images: 'feathrfeaturestore/feathr-registry:nightly' - + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_RBAC_REGISTRY_WEBHOOK }} + + - name: Deploy to Feathr SQL Registry Azure Web App + id: deploy-to-sql-webapp + uses: distributhor/workflow-webhook@v3.0.1 + env: + webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_SQL_REGISTRY_WEBHOOK }} diff --git a/.github/workflows/document-scan.yml b/.github/workflows/document-scan.yml index 3762ca2af..291a04f44 100644 --- a/.github/workflows/document-scan.yml +++ b/.github/workflows/document-scan.yml @@ -1,6 +1,9 @@ name: Feathr Documents' Broken Link Check -on: [push] +on: + push: + branches: [main] + jobs: check-links: runs-on: ubuntu-latest diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 1102d6028..f5d399981 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -22,11 +22,15 @@ on: - "docs/**" - "ui/**" - "**/README.md" + + schedule: + # Runs daily at 1 PM UTC (9 PM CST), will send notification to TEAMS_WEBHOOK + - cron: '00 13 * * *' jobs: sbt_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -41,7 +45,7 @@ jobs: python_lint: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - name: Set up Python 3.8 uses: actions/setup-python@v2 @@ -61,7 +65,7 @@ jobs: databricks_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -87,8 +91,7 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist databricks-cli - python -m pip install -e ./feathr_project/ + python -m pip install -e ./feathr_project/[all] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Set env variable and upload jars env: @@ -132,7 +135,7 @@ jobs: azure_synapse_test: # might be a bit duplication to setup both the azure_synapse test and databricks test, but for now we will keep those to accelerate the test speed runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -166,8 +169,7 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist - python -m pip install -e ./feathr_project/ + python -m pip install -e ./feathr_project/[all] if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run Feathr with Azure Synapse env: @@ -203,7 +205,7 @@ jobs: local_spark_test: runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test')) steps: - uses: actions/checkout@v2 with: @@ -229,9 +231,8 @@ jobs: - name: Install Feathr Package run: | python -m pip install --upgrade pip - python -m pip install pytest pytest-xdist - python -m pip install -e ./feathr_project/ - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install -e ./feathr_project/[all] + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run Feathr with Local Spark env: PROJECT_CONFIG__PROJECT_NAME: "feathr_github_ci_local" @@ -258,4 +259,26 @@ jobs: SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} run: | # skip cloud related tests - pytest feathr_project/test/test_local_spark_e2e.py \ No newline at end of file + pytest feathr_project/test/test_local_spark_e2e.py + + failure_notification: + # If any failure, warning message will be sent + needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] + runs-on: ubuntu-latest + if: failure() && github.event_name == 'schedule' + steps: + - name: Warning + run: | + curl -H 'Content-Type: application/json' -d '{"text": "[WARNING] Daily CI has failure, please check: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }} + + notification: + # Final Daily Report with all job status + needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test] + runs-on: ubuntu-latest + if: always() && github.event_name == 'schedule' + steps: + - name: Get Date + run: echo "NOW=$(date +'%Y-%m-%d')" >> $GITHUB_ENV + - name: Notification + run: | + curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. SBT Test ${{needs.sbt_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }} \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76a01bd06..ec137aa03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,11 @@ Our open source community strives to: - **Be respectful**: We are a world-wide community of professionals, and we conduct ourselves professionally. Disagreement is no excuse for poor behavior and poor manners. - **Understand disagreements**: Disagreements, both social and technical, are useful learning opportunities. Seek to understand the other viewpoints and resolve differences constructively. - **Remember that we’re different**. The strength of our community comes from its diversity, people from a wide range of backgrounds. Different people have different perspectives on issues. Being unable to understand why someone holds a viewpoint doesn’t mean that they’re wrong. Focus on helping to resolve issues and learning from mistakes. +- ## Attribution & Acknowledgements This code of conduct is based on the Open Code of Conduct from the [TODOGroup](https://todogroup.org/blog/open-code-of-conduct/). + +# Committers +Benjamin Le, David Stein, Edwin Cheung, Hangfei Lin, Jimmy Guo, Jinghui Mo, Li Lu, Rama Ramani, Ray Zhang, Xiaoyong Zhu diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..e5808aff2 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,15 @@ +# Component Governance Pipeline +# Runs the Feathr code through Component Governance Detection tool and publishes the result under compliance tab. + +trigger: +- main + +pool: + vmImage: ubuntu-latest + +steps: +- task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' \ No newline at end of file diff --git a/build.sbt b/build.sbt index 2919ddae6..a39db0826 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import sbt.Keys.publishLocalConfiguration ThisBuild / resolvers += Resolver.mavenLocal ThisBuild / scalaVersion := "2.12.15" -ThisBuild / version := "0.8.0" +ThisBuild / version := "0.9.0-rc2" ThisBuild / organization := "com.linkedin.feathr" ThisBuild / organizationName := "linkedin" val sparkVersion = "3.1.3" diff --git a/docs/README.md b/docs/README.md index 1a797ce48..ca67ed446 100644 --- a/docs/README.md +++ b/docs/README.md @@ -159,7 +159,7 @@ Read [Point-in-time Correctness and Point-in-time Join in Feathr](https://feathr ### Running Feathr Examples -Follow the [quick start Jupyter Notebook](./samples/product_recommendation_demo.ipynb) to try it out. There is also a companion [quick start guide](https://feathr-ai.github.io/feathr/quickstart_synapse.html) containing a bit more explanation on the notebook. +Follow the [quick start Jupyter Notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to try it out. There is also a companion [quick start guide](https://feathr-ai.github.io/feathr/quickstart_synapse.html) containing a bit more explanation on the notebook. ## 🗣️ Tech Talks on Feathr diff --git a/docs/concepts/feature-registry.md b/docs/concepts/feature-registry.md index 9bc00b275..e78c0e605 100644 --- a/docs/concepts/feature-registry.md +++ b/docs/concepts/feature-registry.md @@ -74,11 +74,13 @@ client.register_features() all_features = client.list_registered_features(project_name=client.project_name) ``` +Please avoid applying a same name to different features under a certain project. Since it will be treated as updating an exsiting project which is not supported by feathr and will cause errors. + ### Reuse Features from Existing Registry The feature producers can just let the feature consumers know which features exist so the feature consumers can reuse them. For feature consumers, they can reuse existing features from the registry. The whole project can be retrieved to local environment by calling this API `client.get_features_from_registry` with a project name. This encourage feature reuse across organizations. For example, end users of a feature just need to read all feature definitions from the existing projects, then use a few features from the projects and join those features with a new dataset you have. -For example, in the [product recommendation demo notebook](./../samples/product_recommendation_demo.ipynb), some other team members have already defined a few features, such as `feature_user_gift_card_balance` and `feature_user_has_valid_credit_card`. If we want to reuse those features for anti-abuse purpose in a new dataset, what you can do is like this, i.e. just call `get_features_from_registry` to get the features, then put the features you want to query to the anti-abuse dataset you have. +For example, in the [product recommendation demo notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), some other team members have already defined a few features, such as `feature_user_gift_card_balance` and `feature_user_has_valid_credit_card`. If we want to reuse those features for anti-abuse purpose in a new dataset, what you can do is like this, i.e. just call `get_features_from_registry` to get the features, then put the features you want to query to the anti-abuse dataset you have. ```python registered_features_dict = client.get_features_from_registry(client.project_name) diff --git a/docs/concepts/materializing-features.md b/docs/concepts/materializing-features.md index 28d824525..3c31124e2 100644 --- a/docs/concepts/materializing-features.md +++ b/docs/concepts/materializing-features.md @@ -31,6 +31,18 @@ More reference on the APIs: In the above example, we define a Redis table called `nycTaxiDemoFeature` and materialize two features called `f_location_avg_fare` and `f_location_max_fare` to Redis. +## Incremental Aggregation +Use incremental aggregation will significantly expedite the WindowAggTransformation feature calculation. +For example, aggregation sum of a feature F within a 180-day window at day T can be expressed as: F(T) = F(T - 1)+DirectAgg(T-1)-DirectAgg(T - 181). +Once a SNAPSHOT of the first day is generated, the calculation for the following days can leverage it. + +A storeName is required if incremental aggregated is enabled. There could be multiple output Datasets, and each of them need to be stored in a separate folder. The storeName is used as the folder name to create under the base "path". + +Incremental aggregation is enabled by default when using HdfsSink. + +More reference on the APIs: +- [HdfsSink API doc](https://feathr.readthedocs.io/en/latest/feathr.html#feathr.HdfsSink) + ## Feature Backfill It is also possible to backfill the features till a particular time, like below. If the `BackfillTime` part is not specified, it's by default to `now()` (i.e. if not specified, it's equivalent to `BackfillTime(start=now, end=now, step=timedelta(days=1))`). @@ -149,3 +161,53 @@ More reference on the APIs: - [MaterializationSettings API](https://feathr.readthedocs.io/en/latest/feathr.html#feathr.MaterializationSettings) - [HdfsSink API](https://feathr.readthedocs.io/en/latest/feathr.html#feathr.HdfsSource) + +## Expected behavior on Feature Materialization + +When end users materialize features to a sink, what is the expected behavior? + +It seems to be a straightforward question, but actually it is not. Basically when end users want to materialize a feature, Feathr is expecting that: For a certain entity key (say a user_id), there will be multiple features (say user_total_gift_card_balance, and user_purchase_in_last_week). So two checks will be performed: + +1. Those features should have the same entity key (say a user_id). You cannot materialize features for two entity keys in the same materialization job (although you can do it in different jobs), for example materializing `uer_total_purchase` and `product_sold_in_last_week` in the same Feathr materialization job. +2. Those features should all be "aggregated" feature. I.e. they should be a feature which has a type of `WindowAggTransformation`, such as `product_sold_in_last_week`, or `user_latest_total_gift_card_balance`. + +The first constraint is pretty straightforward to explain - since when Feathr materializes certain features, they are used to describe certain aspects of a given entity such as user. Describing `product_sold_in_last_week` would not make sense for users. + +The second constraint is a bit more interesting. For example, you have defined `user_total_gift_card_balance` and it has different value for the same user across different time, say the corresponding value is 40,30,20,20 for the last 4 days, like below. +Original data: + +| UserId | user_total_gift_card_balance | Date | +| ------ | ---------------------------- | ---------- | +| 1 | 40 | 2022/01/01 | +| 1 | 30 | 2022/01/02 | +| 1 | 20 | 2022/01/03 | +| 1 | 20 | 2022/01/04 | +| 2 | 40 | 2022/01/01 | +| 2 | 30 | 2022/01/02 | +| 2 | 20 | 2022/01/03 | +| 2 | 20 | 2022/01/04 | +| 3 | 40 | 2022/01/01 | +| 3 | 30 | 2022/01/02 | +| 3 | 20 | 2022/01/03 | +| 3 | 20 | 2022/01/04 | + +However, the materialized features have no dates associated with them. I.e. the materialized result should be something like this: + +| UserId | user_total_gift_card_balance | +| ------ | ---------------------------- | +| 1 | ? | +| 2 | ? | +| 3 | ? | + +When you ask Feathr to "materialize" `user_total_gift_card_balance` for you, there's only one value that can be materialized, since the materialized feature does not have a date associated with them. So the problem is - for a given `user_id`, only one `user_total_gift_card_balance` can be its feature. Which value you are choosing out of the 4 values? A random value? The latest value? + +It might be natural to think that "we should materialize the latest feature", and that behavior, by definition, is an "aggregation" operation, since we have 4 values for a given `user_id` but we are only materializing and using one of them. In that case, Feathr asks you to explicitly say that you want to materialize the latest feature (i.e. by using [Point-in-time Join](./point-in-time-join.md)) + +```python +feature = Feature(name="user_total_gift_card_balance", + key=UserId, + feature_type=FLOAT, + transform=WindowAggTransformation(agg_expr="gift_card_balance", + agg_func="LATEST", + window="7d")) +``` \ No newline at end of file diff --git a/docs/dev_guide/build-and-push-feathr-registry-docker-image.md b/docs/dev_guide/build-and-push-feathr-registry-docker-image.md index 04b1fe487..873c6a141 100644 --- a/docs/dev_guide/build-and-push-feathr-registry-docker-image.md +++ b/docs/dev_guide/build-and-push-feathr-registry-docker-image.md @@ -76,8 +76,4 @@ docker push feathrfeaturestore/feathr-registry ## Published Feathr Registry Image -The published feathr feature registry is located in [DockerHub here](https://hub.docker.com/r/feathrfeaturestore/feathr-registry). - -## Include the detailed track back info in registry api HTTP error response - -Set environment REGISTRY_DEBUGGING to any non empty string will enable the detailed track back info in registry api http response. This variable is helpful for python client debugging and should only be used for debugging purposes. +The published feathr feature registry is located in [DockerHub here](https://hub.docker.com/r/feathrfeaturestore/feathr-registry). \ No newline at end of file diff --git a/docs/dev_guide/feathr_overall_release_guide.md b/docs/dev_guide/feathr_overall_release_guide.md index 069f6edf4..5d6301a49 100644 --- a/docs/dev_guide/feathr_overall_release_guide.md +++ b/docs/dev_guide/feathr_overall_release_guide.md @@ -10,63 +10,88 @@ This document describes all the release process for the development team. ## Prerequisites -- Make sure the CI tests are passing so there are no surprises on the release day. +- Make sure the CI tests are passing prior to bug bash. - Make sure all the active PRs related to the release are merged. - ## When to Release -- For each major and minor version release, please follow these steps. -- For patch versions, there should be no releases. +The release process is triggered by the release manager. The release manager will decide when to release with following steps: + +1. Ensure Prerequisites are met. +2. Creation of Release Candidate(rc) on GitHub. +3. Bug Bash. +4. Creation of Release on GitHub. +5. Post Release announcement. + +## Release Versioning + +- Major and minor version: X.Y.Z +- Release Candidate: X.Y.Z-rcN ## Writing Release Note Write a release note following past examples [here](https://github.com/feathr-ai/feathr/releases). Read through the [commit log](https://github.com/feathr-ai/feathr/commits/main) to identify the commits after last release to include in the release note. Here are the major things to include -- highlights of the release -- improvements and changes of this release -- new contributors of this release +- Highlights of the release +- Improvements and changes of this release +- New contributors of this release ## Code Changes -Before the release is made, the version needs to be updated in following places + +Before the release candidate or release is made, the version needs to be updated in following places + - [build.sbt](https://github.com/feathr-ai/feathr/blob/main/build.sbt#L3) - For Maven release version -- [setup.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/setup.py#L10) - For PyPi release version +- [version.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathr/version.py#L1) - For Feathr version - [conf.py](https://github.com/feathr-ai/feathr/blob/main/feathr_project/docs/conf.py#L27) - For documentation version -- [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/test/test_user_workspace/feathr_config.yaml#L84) - To set the spark runtime location for Azure Synapse and Azure Databricks used by test suite. -- [constants.py](https://github.com/feathr-ai/feathr/blob/73656fe4a57219e99ff6fede10d51a000ae90fa1/feathr_project/feathr/constants.py#L31) - To set the default maven artifact version +- [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/test/test_user_workspace/feathr_config.yaml#L84) - To set the spark runtime location for Azure Synapse and Azure Databricks used by test suite. Please update all .yaml files under this path. +- [package.json](https://github.com/feathr-ai/feathr/blob/main/ui/package.json#L3) - For Feathr UI version + +Following file should only be updated for release, which means should be skipped for release candidate. + - [azure_resource_provision.json](https://github.com/feathr-ai/feathr/blob/main/docs/how-to-guides/azure_resource_provision.json#L114) - To set the deployment template to pull the latest release image. -## Triggering automated release pipelines -Our goal is to automate the release process as much as possible. So far, we have automated the following steps -1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/docker-publish.yml) to build and publish for our UI and API container to [dockerhub](https://hub.docker.com/r/feathrfeaturestore/feathr-registry/tags). - **Triggers** - Nightly, branch with name pattern "releases/*" +## Release Branches + +Each major and minor release should have a release branch. The release branch should be named as `releases/vX.Y.Z` or `releases/vX.Y.Z-rcN` where `X.Y.Z` is the release version. The release branch should be created from the `main` branch. See past release branches [here](https://github.com/feathr-ai/feathr/branches/all?query=releases). -1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-pypi.yml) for publishing Python package to [PyPi](https://pypi.org/project/feathr/). +## Release Tags - **Triggers** - branch with name pattern "releases/*" +Once the release branch is created, a release tag should be created from the release branch. The release tag should be named as `vX.Y.Z` or `vX.Y.Z-rcN` where `X.Y.Z` is the release version. See past release tags [here](https://github.com/feathr-ai/feathr/tags). -1. Automated Maven workflow - Coming soon. +## Triggering automated release pipelines -**PLEASE NOTE: To trigger the above workflows as part of release, create a new branch with pattern releases/v0.x.0**. See past release branches [here](https://github.com/feathr-ai/feathr/branches/all?query=releases). +Once the release branch and release tag are created, the release pipelines will be triggered automatically. The release pipelines will build the release artifacts and publish them to Maven and PyPI. +1. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/docker-publish.yml) to build and publish for Feathr Registry docker images to [DockerHub](https://hub.docker.com/r/feathrfeaturestore/feathr-registry/tags). -## Release Maven + **Triggers** - Nightly or branch with name pattern "releases/*" + +2. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-pypi.yml) for publishing Python package to [PyPi](https://pypi.org/project/feathr/). -See [Developer Guide for publishing to maven](publish_to_maven.md) + **Triggers** - branch with name pattern "releases/*" + +3. Automated [workflow](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-maven.yml) for publishing the jar to [maven/sonatype repository](https://oss.sonatype.org/). ## Upload Feathr Jar Run the command to generate the Java jar. After the jar is generated, please upload to [Azure storage](https://ms.portal.azure.com/#view/Microsoft_Azure_Storage/ContainerMenuBlade/~/overview/storageAccountId/%2Fsubscriptions%2Fa6c2a7cc-d67e-4a1a-b765-983f08c0423a%2FresourceGroups%2Fazurefeathrintegration%2Fproviders%2FMicrosoft.Storage%2FstorageAccounts%2Fazurefeathrstorage/path/public/etag/%220x8D9E6F64D62D599%22/defaultEncryptionScope/%24account-encryption-key/denyEncryptionScopeOverride//defaultId//publicAccessVal/Container) for faster access. ## Release PyPi -The automated workflow should take care of this, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/publish-to-pypi.yml) to see the triggered run and results. For manual steps, see [Python Package Release Note](https://feathr-ai.github.io/feathr/dev_guide/python_package_release.html) + +The automated workflow should take care of this, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/publish-to-pypi.yml) to see the triggered run and results. For manual steps, see [Python Package Release Guide](https://feathr-ai.github.io/feathr/dev_guide/python_package_release.html) ## Updating docker image for API and Registry + The automated workflow should take care of this as well, you can check under [actions](https://github.com/feathr-ai/feathr/actions/workflows/docker-publish.yml) to see the triggered run and results. For manual steps, see [Feathr Registry docker image](https://feathr-ai.github.io/feathr/dev_guide/build-and-push-feathr-registry-docker-image.html) +## Release Maven + +The automated workflow should take of this too, you can check under [actions](https://github.com/feathr-ai/feathr/blob/main/.github/workflows/publish-to-maven.yml) to see the triggered run and results. For manual steps, see [Feathr Developer Guide for publishing to maven](https://feathr-ai.github.io/feathr/dev_guide/publish_to_maven.html) + ## Testing -Run one of the sample [notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/product_recommendation_demo.ipynb) as it uses the latest package from Maven and PyPi. + +Run one of the sample [notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) as it uses the latest package from Maven and PyPi. ## Announcement diff --git a/docs/how-to-guides/azure-deployment-arm.md b/docs/how-to-guides/azure-deployment-arm.md index 0d833abf0..7bc9a926f 100644 --- a/docs/how-to-guides/azure-deployment-arm.md +++ b/docs/how-to-guides/azure-deployment-arm.md @@ -6,7 +6,7 @@ parent: How-to Guides # Azure Resource Provisioning -The provided Azure Resource Manager (ARM) template deploys the following resources, please make sure you have enough quota in the subscription and region you are deploying this in. You can view your quota and make request on Azure [portal](https://ms.portal.azure.com/#view/Microsoft_Azure_Capacity/QuotaMenuBlade/~/overview) +The provided Azure Resource Manager (ARM) template deploys the following resources, please make sure you have enough quota in the subscription and region you are deploying this in. You can view your quota and make request on Azure [portal](https://ms.portal.azure.com/#view/Microsoft_Azure_Capacity/QuotaMenuBlade/~/overview). 1. Azure Storage account 2. Azure Purview (metadata store if you selected Azure-Purview as registry backend) @@ -17,9 +17,9 @@ The provided Azure Resource Manager (ARM) template deploys the following resourc 7. Azure Event Hub 8. Azure Redis -Please note, you need to have **owner access** in the resource group you are deploying this in. Owner access is required to assign role to managed identity within ARM template so it can access key vault and store secrets. If you don't have such permission, you might want to contact your IT admin to see if they can do that. +### Please Note: you need to have the **Owner Role** in the resource group you are deploying this in. Owner access is required to assign role to managed identity within the ARM template so it can access key vault and store secrets. It is also required by the permission section in our sample notebooks. If you don't have such permission, you might want to contact your IT admin to see if they can do that. -Although we recommend end users deploy the resources using the ARM template, we understand that in many situations where users want to reuse existing resources instead of creating new resources; or users have many other permission issues. See [Manually connecting existing resources](#manually-connecting-existing-resources) for more details. +Although we recommend end users deploy the resources using the ARM template, we understand that in many situations where users want to reuse existing resources instead of creating new resources; or users may have permission issues. See [Manually connecting existing resources](#manually-connecting-existing-resources) for more details. ## Architecture @@ -34,10 +34,12 @@ Feathr has native cloud integration and getting started with Feathr is very stra The very first step is to create an Azure Active Directory (AAD) application to enable authentication on the Feathr UI (which gets created as part of the deployment script). Currently it is not possible to create one through ARM template but you can easily create one by running the following CLI commands in the [Cloud Shell](https://shell.azure.com/bash). +### Please make note of the Client ID and Tenant ID for the AAD app, you will need it in the ARM template deployment section. + ```bash # This is the prefix you want to name your resources with, make a note of it, you will need it during deployment. # Note: please keep the `resourcePrefix` short (less than 15 chars), since some of the Azure resources need the full name to be less than 24 characters. Only lowercase alphanumeric characters are allowed for resource prefix. -resource_prefix="userprefix1" +resource_prefix="yourprefix" # Please don't change this name, a corresponding webapp with same name gets created in subsequent steps. sitename="${resource_prefix}webapp" diff --git a/docs/how-to-guides/azure-deployment-cli.md b/docs/how-to-guides/azure-deployment-cli.md index 3762f7b3f..70067b148 100644 --- a/docs/how-to-guides/azure-deployment-cli.md +++ b/docs/how-to-guides/azure-deployment-cli.md @@ -117,7 +117,6 @@ echo "AZURE_TENANT_ID: $sp_tenantid" echo "AZURE_CLIENT_SECRET: $sp_password" This will give three variables: AZURE_CLIENT_ID, AZURE_TENANT_ID and AZURE_CLIENT_SECRET. You will need them later. ``` - Note: **You should save AZURE_CLIENT_SECRET because you will only see it once here** ## Create a storage account diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index e52ca3706..1a9e15dac 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -90,17 +90,17 @@ feathr_client = FeathrClient(..., secret_manager_client = cache) | AZURE_CLIENT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | | AZURE_TENANT_ID | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | | AZURE_CLIENT_SECRET | Client ID for authentication into Azure Services. Read [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.environmentcredential?view=azure-python) for more details. | This is required if you are using Service Principal to login with Feathr. | -| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. | Optional | +| OFFLINE_STORE__ADLS__ADLS_ENABLED | Whether to enable ADLS as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | | ADLS_ACCOUNT | ADLS account that you connect to. | Required if using ADLS as an offline store. | | ADLS_KEY | ADLS key that you connect to. | Required if using ADLS as an offline store. | -| OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. | +| OFFLINE_STORE__WASB__WASB_ENABLED | Whether to enable Azure BLOB storage as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | | WASB_ACCOUNT | Azure BLOB Storage account that you connect to. | Required if using Azure BLOB Storage as an offline store. | | WASB_KEY | Azure BLOB Storage key that you connect to. | Required if using Azure BLOB Storage as an offline store. | | S3_ACCESS_KEY | AWS S3 access key for the S3 account. | Required if using AWS S3 Storage as an offline store. | | S3_SECRET_KEY | AWS S3 secret key for the S3 account. | Required if using AWS S3 Storage as an offline store. | -| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. | Optional | +| OFFLINE_STORE__S3__S3_ENABLED | Whether to enable S3 as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | | OFFLINE_STORE__S3__S3_ENDPOINT | S3 endpoint. If you use S3 endpoint, then you need to provide access key and secret key in the environment variable as well. | Required if using AWS S3 Storage as an offline store. | -| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. | Optional | +| OFFLINE_STORE__JDBC__JDBC_ENABLED | Whether to enable JDBC as offline store or not. Available value: "True" or "False". Equivalent to "False" if not set. | Optional | | OFFLINE_STORE__JDBC__JDBC_DATABASE | If using JDBC endpoint as offline store, this config specifies the JDBC database to read from. | Required if using JDBC sources as offline store | | OFFLINE_STORE__JDBC__JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from. Same as `JDBC_TABLE`. | Required if using JDBC sources as offline store | | JDBC_TABLE | If using JDBC endpoint as offline store, this config specifies the JDBC table to read from | Required if using JDBC sources as offline store | @@ -108,6 +108,7 @@ feathr_client = FeathrClient(..., secret_manager_client = cache) | JDBC_PASSWORD | If using JDBC endpoint as offline store, this config specifies the JDBC password | Required if using JDBC sources as offline store | | KAFKA_SASL_JAAS_CONFIG | see [here](#KAFKA_SASL_JAAS_CONFIG) for more details. | Required if using Kafka/EventHub as streaming source input. | | PROJECT_CONFIG__PROJECT_NAME | Configures the project name. | Required | +| OFFLINE_STORE__SNOWFLAKE__SNOWFLAKE_ENABLED | Configures whether Snowflake as offline store is enabled or not. Available value: "True" or "False". Equivalent to "False" if not set. | Required if using Snowflake as an offline store. | | OFFLINE_STORE__SNOWFLAKE__URL | Configures the Snowflake URL. Usually it's something like `dqllago-ol19457.snowflakecomputing.com`. | Required if using Snowflake as an offline store. | | OFFLINE_STORE__SNOWFLAKE__USER | Configures the Snowflake user. | Required if using Snowflake as an offline store. | | OFFLINE_STORE__SNOWFLAKE__ROLE | Configures the Snowflake role. Usually it's something like `ACCOUNTADMIN`. | Required if using Snowflake as an offline store. | @@ -116,7 +117,7 @@ feathr_client = FeathrClient(..., secret_manager_client = cache) | SPARK_CONFIG__SPARK_RESULT_OUTPUT_PARTS | Configure number of parts for the spark output for feature generation job | Required | | SPARK_CONFIG__AZURE_SYNAPSE__DEV_URL | Dev URL to the synapse cluster. Usually it's something like `https://yourclustername.dev.azuresynapse.net` | Required if using Azure Synapse | | SPARK_CONFIG__AZURE_SYNAPSE__POOL_NAME | name of the spark pool that you are going to use | Required if using Azure Synapse | -| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here | Required if using Azure Synapse | +| SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR | A location that Synapse has access to. This workspace dir stores all the required configuration files and the jar resources. All the feature definitions will be uploaded here. Suggest to use an empty dir for a new spark job to avoid conflicts. | Required if using Azure Synapse | | SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_SIZE | Specifies the executor size for the Azure Synapse cluster. Currently the options are `Small`, `Medium`, `Large`. | Required if using Azure Synapse | | SPARK_CONFIG__AZURE_SYNAPSE__EXECUTOR_NUM | Specifies the number of executors for the Azure Synapse cluster | Required if using Azure Synapse | | SPARK_CONFIG__AZURE_SYNAPSE__FEATHR_RUNTIME_LOCATION | Specifies the Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `abfss:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Azure Synapse | @@ -124,14 +125,15 @@ feathr_client = FeathrClient(..., secret_manager_client = cache) | SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE | Config string including run time information, spark version, machine size, etc. See [below](#SPARK_CONFIG__DATABRICKS__CONFIG_TEMPLATE) for more details. | Required if using Databricks | | SPARK_CONFIG__DATABRICKS__WORK_DIR | Workspace dir for storing all the required configuration files and the jar resources. All the feature definitions will be uploaded here. | Required if using Databricks | | SPARK_CONFIG__DATABRICKS__FEATHR_RUNTIME_LOCATION | Feathr runtime location. Support local paths, path start with `http(s)://`, and paths start with `dbfs:/`. If not set, will use the [Feathr package published in Maven](https://search.maven.org/artifact/com.linkedin.feathr/feathr_2.12). | Required if using Databricks | +| DATABRICKS_WORKSPACE_TOKEN_VALUE | Token value to access databricks workspace. More details can be found at [Authentication using Databricks personal access tokens](https://docs.databricks.com/dev-tools/api/latest/authentication.html) | Required if using Databricks | | ONLINE_STORE__REDIS__HOST | Redis host name to access Redis cluster. | Required if using Redis as online store. | | ONLINE_STORE__REDIS__PORT | Redis port number to access Redis cluster. | Required if using Redis as online store. | | ONLINE_STORE__REDIS__SSL_ENABLED | Whether SSL is enabled to access Redis cluster. | Required if using Redis as online store. | | REDIS_PASSWORD | Password for the Redis cluster. | Required if using Redis as online store. | | FEATURE_REGISTRY__API_ENDPOINT | Specifies registry endpoint. | Required if using registry service. | -| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__DELIMITER | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| -| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION | Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__PURVIEW_NAME (Deprecated Soon) | Configure the name of the purview endpoint. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__DELIMITER (Deprecated Soon) | See [here](#FEATURE_REGISTRY__PURVIEW__DELIMITER) for more details. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| +| FEATURE_REGISTRY__PURVIEW__TYPE_SYSTEM_INITIALIZATION (Deprecated Soon)| Controls whether the type system (think this as the "schema" for the registry) will be initialized or not. Usually this is only required to be set to `True` to initialize schema, and then you can set it to `False` to shorten the initialization time. | Required if using Purview directly without registry service. Deprecate soon, see [here](#deprecation) for more details.| # Explanation for selected configurations diff --git a/docs/how-to-guides/local-spark-provider.md b/docs/how-to-guides/local-spark-provider.md index 433af64f3..7f63fb042 100644 --- a/docs/how-to-guides/local-spark-provider.md +++ b/docs/how-to-guides/local-spark-provider.md @@ -36,7 +36,7 @@ A spark-submit script will auto generated in your workspace under `debug` folder spark-submit \ --master local[*] \ --name project_feathr_local_spark_test \ - --packages "org.apache.spark:spark-avro_2.12:3.3.0,com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8,com.microsoft.azure:spark-mssql-connector_2.12:1.2.0,org.apache.logging.log4j:log4j-core:2.17.2,com.typesafe:config:1.3.4,com.fasterxml.jackson.core:jackson-databind:2.12.6.1,org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7,org.apache.hadoop:hadoop-common:2.7.7,org.apache.avro:avro:1.8.2,org.apache.xbean:xbean-asm6-shaded:4.10,org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3,com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21,org.apache.kafka:kafka-clients:3.1.0,com.google.guava:guava:31.1-jre,it.unimi.dsi:fastutil:8.1.1,org.mvel:mvel2:2.2.8.Final,com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.3,com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.6,com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.6,com.jasonclawson:jackson-dataformat-hocon:1.1.0,com.redislabs:spark-redis_2.12:3.1.0,org.apache.xbean:xbean-asm6-shaded:4.10,com.google.protobuf:protobuf-java:3.19.4,net.snowflake:snowflake-jdbc:3.13.18,net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2,org.apache.commons:commons-lang3:3.12.0,org.xerial:sqlite-jdbc:3.36.0.3,com.github.changvvb:jackson-module-caseclass_2.12:1.1.1,com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.11.1,org.eclipse.jetty:jetty-util:9.3.24.v20180605,commons-io:commons-io:2.6,org.apache.hadoop:hadoop-azure:2.7.4,com.microsoft.azure:azure-storage:8.6.4,com.linkedin.feathr:feathr_2.12:0.8.0" \ + --packages "org.apache.spark:spark-avro_2.12:3.3.0,com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8,com.microsoft.azure:spark-mssql-connector_2.12:1.2.0,org.apache.logging.log4j:log4j-core:2.17.2,com.typesafe:config:1.3.4,com.fasterxml.jackson.core:jackson-databind:2.12.6.1,org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7,org.apache.hadoop:hadoop-common:2.7.7,org.apache.avro:avro:1.8.2,org.apache.xbean:xbean-asm6-shaded:4.10,org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3,com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21,org.apache.kafka:kafka-clients:3.1.0,com.google.guava:guava:31.1-jre,it.unimi.dsi:fastutil:8.1.1,org.mvel:mvel2:2.2.8.Final,com.fasterxml.jackson.module:jackson-module-scala_2.12:2.13.3,com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.6,com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.12.6,com.jasonclawson:jackson-dataformat-hocon:1.1.0,com.redislabs:spark-redis_2.12:3.1.0,org.apache.xbean:xbean-asm6-shaded:4.10,com.google.protobuf:protobuf-java:3.19.4,net.snowflake:snowflake-jdbc:3.13.18,net.snowflake:spark-snowflake_2.12:2.10.0-spark_3.2,org.apache.commons:commons-lang3:3.12.0,org.xerial:sqlite-jdbc:3.36.0.3,com.github.changvvb:jackson-module-caseclass_2.12:1.1.1,com.azure.cosmos.spark:azure-cosmos-spark_3-1_2-12:4.11.1,org.eclipse.jetty:jetty-util:9.3.24.v20180605,commons-io:commons-io:2.6,org.apache.hadoop:hadoop-azure:2.7.4,com.microsoft.azure:azure-storage:8.6.4,com.linkedin.feathr:feathr_2.12:0.9.0-rc2" \ --conf "spark.driver.extraClassPath=../target/scala-2.12/classes:jars/config-1.3.4.jar:jars/jackson-dataformat-hocon-1.1.0.jar:jars/jackson-module-caseclass_2.12-1.1.1.jar:jars/mvel2-2.2.8.Final.jar:jars/fastutil-8.1.1.jar" \ --conf "spark.hadoop.fs.wasbs.impl=org.apache.hadoop.fs.azure.NativeAzureFileSystem" \ --class com.linkedin.feathr.offline.job.FeatureJoinJob \ diff --git a/docs/quickstart_databricks.md b/docs/quickstart_databricks.md index dff5b5f0f..30eaaa835 100644 --- a/docs/quickstart_databricks.md +++ b/docs/quickstart_databricks.md @@ -5,13 +5,13 @@ title: Quick Start Guide with Databricks # Feathr Quick Start Guide with Databricks -For Databricks, you can simply upload [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb) to your Databricks cluster and just run it in the Databricks cluster. It has been pre-configured to use the current Databricks cluster to submit jobs. +For Databricks, you can simply upload [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb) to your Databricks cluster and just run it in the Databricks cluster. It has been pre-configured to use the current Databricks cluster to submit jobs. 1. Import Notebooks in your Databricks cluster: ![Import Notebooks](./images/databricks_quickstart1.png) -2. Paste the [link to Databricks getting started notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb): +2. Paste the [link to Databricks getting started notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb): ![Import Notebooks](./images/databricks_quickstart2.png) @@ -21,7 +21,7 @@ For Databricks, you can simply upload [this notebook](./samples/databricks/datab Although Databricks Notebooks are great tools, there are also large developer communities that prefer the usage of Visual Studio Code, where [it has native support for Python and Jupyter Notebooks](https://code.visualstudio.com/docs/datascience/jupyter-notebooks) with many great features such as syntax highlight and IntelliSense. -In [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb), there are a few lines of code like this: +In [this notebook](./samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb), there are a few lines of code like this: ```python # Get current databricks notebook context diff --git a/docs/quickstart_synapse.md b/docs/quickstart_synapse.md index 5dee17931..d07198d92 100644 --- a/docs/quickstart_synapse.md +++ b/docs/quickstart_synapse.md @@ -43,7 +43,7 @@ pip install git+https://github.com/feathr-ai/feathr.git#subdirectory=feathr_proj ## Step 3: Run the sample notebook -We've provided a self-contained [sample notebook](./samples/product_recommendation_demo.ipynb) to act as the main content of this getting started guide. This documentation should be used more like highlights and further explanations of that demo notebook. +We've provided a self-contained [sample notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to act as the main content of this getting started guide. This documentation should be used more like highlights and further explanations of that demo notebook. ## Step 4: Update Feathr config @@ -61,9 +61,6 @@ project_config: # Redis password for your online store - "REDIS_PASSWORD" # Client IDs and Client Secret for the service principal. Read the getting started docs on how to get those information. - - "AZURE_CLIENT_ID" - - "AZURE_TENANT_ID" - - "AZURE_CLIENT_SECRET" offline_store: --- @@ -91,16 +88,13 @@ os.environ['ONLINE_STORE__REDIS__HOST'] = 'feathrazure.redis.cache.windows.net' ## Step 5: Setup environment variables -In the self-contained [sample notebook](./samples/product_recommendation_demo.ipynb), you also have to setup a few environment variables like below in order to access those cloud resources. You should be able to get those values from the first step. +In the self-contained [sample notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), you also have to setup a few environment variables like below in order to access those cloud resources. You should be able to get those values from the first step. These values can also be retrieved by using cloud key value store, such as [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/): ```python import os os.environ['REDIS_PASSWORD'] = '' -os.environ['AZURE_CLIENT_ID'] = '' -os.environ['AZURE_TENANT_ID'] = '' -os.environ['AZURE_CLIENT_SECRET'] = '' ``` Please refer to [A note on using azure key vault to store credentials](https://github.com/feathr-ai/feathr/blob/41e7496b38c43af6d7f8f1de842f657b27840f6d/docs/how-to-guides/feathr-configuration-and-env.md#a-note-on-using-azure-key-vault-to-store-credentials) for more details. @@ -187,7 +181,7 @@ client.multi_get_online_features("nycTaxiDemoFeature", ["239", "265"], ['f_locat ## Next steps -- Run the [demo notebook](./samples/product_recommendation_demo.ipynb) to understand the workflow of Feathr. +- Run the [demo notebook](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb) to understand the workflow of Feathr. - Read the [Feathr Documentation Page](https://feathr-ai.github.io/feathr/) page to understand the Feathr abstractions. - Read guide to understand [how to setup Feathr on Azure using Azure Resource Manager template](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html). - Read guide to understand [how to setup Feathr step by step on Azure using Azure CLI](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html). diff --git a/docs/samples/product_recommendation_demo.ipynb b/docs/samples/azure_synapse/product_recommendation_demo.ipynb similarity index 68% rename from docs/samples/product_recommendation_demo.ipynb rename to docs/samples/azure_synapse/product_recommendation_demo.ipynb index aa7699eb5..dc6eddf88 100644 --- a/docs/samples/product_recommendation_demo.ipynb +++ b/docs/samples/azure_synapse/product_recommendation_demo.ipynb @@ -4,38 +4,42 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Demo Notebook: Feathr Feature Store on Azure\n", + "# Product Recommendation with Feathr on Azure\n", "\n", "This notebook demonstrates how Feathr Feature Store can simplify and empower your model training and inference. You will learn:\n", "\n", "1. Define sharable features using Feathr API\n", - "2. Create a training dataset via point-in-time feature join with Feathr API\n", - "3. Materialize features to online store and then retrieve them with Feathr API" + "2. Register features with register API.\n", + "3. Create a training dataset via point-in-time feature join with Feathr API\n", + "4. Materialize features to online store and then retrieve them with Feathr API\n", + "\n", + "In this tutorial, we use Feathr to create a model that predicts users' product rating. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Prerequisite: Use Quick Start Template to Provision Azure Resources\n", + "## 1. Prerequisite: Use Azure Resource Manager(ARM) to Provision Azure Resources\n", "\n", "First step is to provision required cloud resources if you want to use Feathr. Feathr provides a python based client to interact with cloud resources.\n", "\n", - "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. Due to the complexity of the possible cloud environment, it is almost impossible to create a script that works for all the use cases. Because of this, [azure_resource_provision.sh](https://github.com/linkedin/feathr/blob/main/docs/how-to-guides/azure_resource_provision.sh) is a full end to end command line to create all the required resources, and you can tailor the script as needed, while [the companion documentation](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html) can be used as a complete guide for using that shell script. \n", + "Please follow the steps [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html) to provision required cloud resources. This will create a new resource group and deploy the needed Azure resources in it. \n", "\n", + "If you already have an existing resource group and only want to install few resources manually you can refer to the cli documentation [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html). It provides CLI commands to install the needed resources. \n", + "**Please Note: CLI documentation is for advance users since there are lot of configurations and role assignment that would have to be done manually so it won't work out of box and should just be used for reference. ARM template is the preferred way to deploy.**\n", "\n", - "![Architecture](https://github.com/linkedin/feathr/blob/main/docs/images/architecture.png?raw=true)" + "The below architecture diagram represents how different resources interact with each other\n", + "![Architecture](https://github.com/feathr-ai/feathr/blob/main/docs/images/architecture.png?raw=true)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Prerequisite: Install Feathr and Import Dependencies\n", - "\n", - "Install Feathr using pip:\n", + "## 2. Prerequisite: Login to Azure and Install Feathr\n", "\n", - "`pip install -U feathr pandavro scikit-learn`" + "Login to Azure with a device code (You will see instructions in the output once you execute the cell):" ] }, { @@ -44,53 +48,14 @@ "metadata": {}, "outputs": [], "source": [ - "# Import Dependencies\n", - "import glob\n", - "import os\n", - "import tempfile\n", - "from datetime import datetime, timedelta\n", - "from math import sqrt\n", - "\n", - "import pandas as pd\n", - "import pandavro as pdx\n", - "from feathr import FeathrClient\n", - "from feathr import BOOLEAN, FLOAT, INT32, ValueType\n", - "from feathr import Feature, DerivedFeature, FeatureAnchor\n", - "from feathr import BackfillTime, MaterializationSettings\n", - "from feathr import FeatureQuery, ObservationSettings\n", - "from feathr import RedisSink\n", - "from feathr import INPUT_CONTEXT, HdfsSource\n", - "from feathr import WindowAggTransformation\n", - "from feathr import TypedKey\n", - "from sklearn.metrics import mean_squared_error\n", - "from sklearn.model_selection import train_test_split\n", - "from azure.identity import DefaultAzureCredential\n", - "from azure.keyvault.secrets import SecretClient" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Configure the required environment with Feathr Quick Start Template\n", - "\n", - "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. Run the code below to install Feathr, login to Azure to get the required credentials to access more cloud resources." + "! az login --use-device-code" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "**REQUIRED STEP: Fill in the resource prefix when provisioning the resources**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "resource_prefix = \"feathr_resource_prefix\"" + "Install Feathr and dependencies to run this notebook." ] }, { @@ -99,14 +64,14 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install feathr azure-cli pandavro scikit-learn" + "%pip install -U feathr pandavro scikit-learn" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Login to Azure with a device code (You will see instructions in the output):" + "Import Dependencies to make sure everything is installed correctly" ] }, { @@ -115,7 +80,27 @@ "metadata": {}, "outputs": [], "source": [ - "! az login --use-device-code" + "import glob\n", + "import os\n", + "import tempfile\n", + "from datetime import datetime, timedelta\n", + "from math import sqrt\n", + "\n", + "import pandas as pd\n", + "import pandavro as pdx\n", + "from feathr import FeathrClient\n", + "from feathr import BOOLEAN, FLOAT, INT32, ValueType\n", + "from feathr import Feature, DerivedFeature, FeatureAnchor\n", + "from feathr import BackfillTime, MaterializationSettings\n", + "from feathr import FeatureQuery, ObservationSettings\n", + "from feathr import RedisSink\n", + "from feathr import INPUT_CONTEXT, HdfsSource\n", + "from feathr import WindowAggTransformation\n", + "from feathr import TypedKey\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from azure.identity import AzureCliCredential\n", + "from azure.keyvault.secrets import SecretClient" ] }, { @@ -123,20 +108,19 @@ "metadata": {}, "source": [ "\n", - "**Permission**\n", + "## 3. Prerequisite: Set the required permissions\n", "\n", - "To proceed with the following steps, you may need additional permission: permission to access the keyvault, permission to access the Storage Blob as a Contributor and permission to submit jobs to Synapse cluster. Skip this step if you have already given yourself the access. Otherwise, run the following lines of command in the Cloud Shell before running the cell below.\n", + "Before you proceed further, you would need additional permissions: permission to access the keyvault, permission to access the Storage Blob as a Contributor and permission to submit jobs to Synapse cluster. Run the following lines of command in the [Cloud Shell](https://shell.azure.com) before running the cells below. Please replace the resource_prefix with the prefix you used in ARM template deployment.\n", "\n", "```\n", - "userId=\n", - "resource_prefix=\n", - "synapse_workspace_name=\"${resource_prefix}syws\"\n", - "keyvault_name=\"${resource_prefix}kv\"\n", - "objectId=$(az ad user show --id $userId --query id -o tsv)\n", - "az keyvault update --name $keyvault_name --enable-rbac-authorization false\n", - "az keyvault set-policy -n $keyvault_name --secret-permissions get list --object-id $objectId\n", - "az role assignment create --assignee $userId --role \"Storage Blob Data Contributor\"\n", - "az synapse role assignment create --workspace-name $synapse_workspace_name --role \"Synapse Contributor\" --assignee $userId\n", + " resource_prefix=\"YOUR_RESOURCE_PREFIX\"\n", + " synapse_workspace_name=\"${resource_prefix}syws\"\n", + " keyvault_name=\"${resource_prefix}kv\"\n", + " objectId=$(az ad signed-in-user show --query id -o tsv)\n", + " az keyvault update --name $keyvault_name --enable-rbac-authorization false\n", + " az keyvault set-policy -n $keyvault_name --secret-permissions get list --object-id $objectId\n", + " az role assignment create --assignee $userId --role \"Storage Blob Data Contributor\"\n", + " az synapse role assignment create --workspace-name $synapse_workspace_name --role \"Synapse Contributor\" --assignee $userId\n", "```\n" ] }, @@ -144,7 +128,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Get all the required credentials from Azure KeyVault" + "# 4. Prerequisite: Feathr Configuration\n", + "\n", + "### Setting the environment variables\n", + "Set the environment variables that will be used by Feathr as configuration. Feathr supports configuration via enviroment variables and yaml, you can read more about it [here](https://feathr-ai.github.io/feathr/how-to-guides/feathr-configuration-and-env.html).\n", + "\n", + "**Fill in the `resource_prefix` that you used while provisioning the resources in Step 1 using ARM.**" ] }, { @@ -153,44 +142,49 @@ "metadata": {}, "outputs": [], "source": [ - "# Get all the required credentials from Azure Key Vault\n", - "key_vault_name=resource_prefix+\"kv\"\n", - "synapse_workspace_url=resource_prefix+\"syws\"\n", - "adls_account=resource_prefix+\"dls\"\n", - "adls_fs_name=resource_prefix+\"fs\"\n", - "purview_name=resource_prefix+\"purview\"\n", - "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", - "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", - "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", - "retrieved_secret = client.get_secret(secretName).value\n", - "\n", - "# Get redis credentials; This is to parse Redis connection string.\n", - "redis_port=retrieved_secret.split(',')[0].split(\":\")[1]\n", - "redis_host=retrieved_secret.split(',')[0].split(\":\")[0]\n", - "redis_password=retrieved_secret.split(',')[1].split(\"password=\",1)[1]\n", - "redis_ssl=retrieved_secret.split(',')[2].split(\"ssl=\",1)[1]\n", - "\n", - "# Set the resource link\n", - "os.environ['spark_config__azure_synapse__dev_url'] = f'https://{synapse_workspace_url}.dev.azuresynapse.net'\n", - "os.environ['spark_config__azure_synapse__pool_name'] = 'spark31'\n", - "os.environ['spark_config__azure_synapse__workspace_dir'] = f'abfss://{adls_fs_name}@{adls_account}.dfs.core.windows.net/feathr_project'\n", - "os.environ['online_store__redis__host'] = redis_host\n", - "os.environ['online_store__redis__port'] = redis_port\n", - "os.environ['online_store__redis__ssl_enabled'] = redis_ssl\n", - "os.environ['REDIS_PASSWORD']=redis_password\n", - "feathr_output_path = f'abfss://{adls_fs_name}@{adls_account}.dfs.core.windows.net/feathr_output'" + "RESOURCE_PREFIX = \"YOUR_RESOURCE_PREFIX\" # from ARM deployment in Step 1\n", + "FEATHR_PROJECT_NAME=\"YOUR_PROJECT_NAME\" # provide a unique name" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "## Prerequisite: Configure the required environment (Skip this step if using the above Quick Start Template)\n", "\n", - "In the first step (Provision cloud resources), you should have provisioned all the required cloud resources. If you use Feathr CLI to create a workspace, you should have a folder with a file called `feathr_config.yaml` in it with all the required configurations. Otherwise, update the configuration below.\n", + "# Get name for deployed resources using the resource prefix\n", + "KEY_VAULT_NAME=f\"{RESOURCE_PREFIX}kv\"\n", + "SYNAPSE_WORKSPACE_NAME=f\"{RESOURCE_PREFIX}syws\"\n", + "ADLS_ACCOUNT=f\"{RESOURCE_PREFIX}dls\"\n", + "ADLS_FS_NAME=f\"{RESOURCE_PREFIX}fs\"\n", + "KEY_VAULT_URI = f\"https://{KEY_VAULT_NAME}.vault.azure.net\"\n", + "FEATHR_API_APP = f\"{RESOURCE_PREFIX}webapp\"\n", + "\n", "\n", - "The code below will write this configuration string to a temporary location and load it to Feathr. Please still refer to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It should also have more explanations on the meaning of each variable." + "# Getting the credential object for Key Vault client\n", + "credential = AzureCliCredential()\n", + "client = SecretClient(vault_url=KEY_VAULT_URI, credential=credential)\n", + "\n", + "# Getting Redis store's connection string.\n", + "retrieved_secret = client.get_secret(\"FEATHR-ONLINE-STORE-CONN\").value\n", + "\n", + "# Parse Redis connection string\n", + "REDIS_PORT=retrieved_secret.split(',')[0].split(\":\")[1]\n", + "REDIS_HOST=retrieved_secret.split(',')[0].split(\":\")[0]\n", + "REDIS_PASSWORD=retrieved_secret.split(',')[1].split(\"password=\",1)[1]\n", + "REDIS_SSL=retrieved_secret.split(',')[2].split(\"ssl=\",1)[1]\n", + "# Set password as environment variable.\n", + "os.environ['REDIS_PASSWORD']=REDIS_PASSWORD" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Write the configuration as yaml file.\n", + "\n", + "The code below will write this configuration string to a temporary location and load it to Feathr. Please refer to [feathr_config.yaml](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for full list of configuration options and details about them." ] }, { @@ -200,68 +194,38 @@ "outputs": [], "source": [ "import tempfile\n", - "yaml_config = \"\"\"\n", - "# Please refer to https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml for explanations on the meaning of each field.\n", + "yaml_config = f\"\"\"\n", "api_version: 1\n", "project_config:\n", - " project_name: 'feathr_getting_started'\n", - " required_environment_variables:\n", - " - 'REDIS_PASSWORD'\n", + " project_name: '{FEATHR_PROJECT_NAME}'\n", "offline_store:\n", "# Please set 'enabled' flags as true (false by default) if any of items under the same paths are expected to be visited\n", " adls:\n", " adls_enabled: true\n", " wasb:\n", " wasb_enabled: true\n", - " s3:\n", - " s3_enabled: false\n", - " s3_endpoint: 's3.amazonaws.com'\n", - " jdbc:\n", - " jdbc_enabled: false\n", - " jdbc_database: 'feathrtestdb'\n", - " jdbc_table: 'feathrtesttable'\n", - " snowflake:\n", - " snowflake_enabled: false\n", - " url: \".snowflakecomputing.com\"\n", - " user: \"\"\n", - " role: \"\"\n", "spark_config:\n", " spark_cluster: 'azure_synapse'\n", " spark_result_output_parts: '1'\n", " azure_synapse:\n", - " dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net'\n", - " pool_name: 'spark3'\n", - " workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_getting_started'\n", + " dev_url: 'https://{SYNAPSE_WORKSPACE_NAME}.dev.azuresynapse.net'\n", + " pool_name: 'spark31'\n", + " workspace_dir: 'abfss://{ADLS_FS_NAME}@{ADLS_ACCOUNT}.dfs.core.windows.net/feathr_project'\n", " executor_size: 'Small'\n", " executor_num: 1\n", - " databricks:\n", - " workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net'\n", - " config_template: {'run_name':'','new_cluster':{'spark_version':'9.1.x-scala2.12','node_type_id':'Standard_D3_v2','num_workers':2,'spark_conf':{}},'libraries':[{'jar':''}],'spark_jar_task':{'main_class_name':'','parameters':['']}}\n", - " work_dir: 'dbfs:/feathr_getting_started'\n", "online_store:\n", " redis:\n", - " host: 'feathrazuretest3redis.redis.cache.windows.net'\n", - " port: 6380\n", - " ssl_enabled: True\n", + " host: '{REDIS_HOST}'\n", + " port: {REDIS_PORT}\n", + " ssl_enabled: {REDIS_SSL}\n", "feature_registry:\n", - " api_endpoint: \"https://feathr-sql-registry.azurewebsites.net/api/v1\"\n", + " api_endpoint: 'https://{FEATHR_API_APP}.azurewebsites.net/api/v1'\n", "\"\"\"\n", + "\n", "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", "with open(tmp.name, \"w\") as text_file:\n", " text_file.write(yaml_config)\n", - "feathr_output_path = f'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_output'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Prerequisite: Setup necessary environment variables (Skip this step if using the above Quick Start Template)\n", - "\n", - "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", - "\n", - "To run this notebook, for Azure users, you need AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET and REDIS_PASSWORD.\n", - "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." + "feathr_output_path = f'abfss://{ADLS_FS_NAME}@{ADLS_ACCOUNT}.dfs.core.windows.net/feathr_output'" ] }, { @@ -270,10 +234,10 @@ "source": [ "# Define sharable features using Feathr API\n", "\n", - "In this tutorial, we use Feathr Feature Store to help create a model that predicts users product rating. To make it simple, let's just predict users' rating for ONE product for an e-commerce website. (We have an [advanced demo](./product_recommendation_demo_advanced.ipynb) that predicts ratings for arbitrary products.)\n", + "In this tutorial, we use Feathr Feature Store and create a model that predicts users' product rating. To make it simple, let's just predict users' rating for ONE product for an e-commerce website. (We have an [advanced demo](../product_recommendation_demo_advanced.ipynb) that predicts ratings for arbitrary products.)\n", "\n", "\n", - "## Initialize Feathr Client\n", + "### Initialize Feathr Client\n", "\n", "Let's initialize a Feathr client first. The Feathr client provides all the APIs we need to interact with Feathr Feature Store." ] @@ -284,14 +248,14 @@ "metadata": {}, "outputs": [], "source": [ - "feathr_client = FeathrClient(config_path=tmp.name)" + "feathr_client = FeathrClient(config_path=tmp.name, credential=credential)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Understand the Raw Datasets\n", + "### Understand the Raw Datasets\n", "We have 3 raw datasets to work with: one observation dataset(a.k.a. label dataset) and two raw datasets to generate features." ] }, @@ -305,6 +269,7 @@ "# Observation dataset usually comes with a event_timestamp to denote when the observation happened.\n", "# The label here is product_rating. Our model objective is to predict a user's rating for this product.\n", "import pandas as pd\n", + "# Public URL hosting mock data\n", "pd.read_csv(\"https://azurefeathrstorage.blob.core.windows.net/public/sample_data/product_recommendation_sample/user_observation_mock_data.csv\")" ] }, @@ -339,14 +304,14 @@ " After a bit of data exploration, we want to create a training dataset like this:\n", "\n", " \n", - "![Feature Flow](https://github.com/linkedin/feathr/blob/main/docs/images/product_recommendation.jpg?raw=true)" + "![Feature Flow](https://github.com/feathr-ai/feathr/blob/main/docs/images/product_recommendation.jpg?raw=true)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## What's a Feature in Feathr\n", + "### What's a Feature in Feathr\n", "A feature is an individual measurable property or characteristic of a phenomenon which is sometimes time-sensitive. \n", "\n", "In Feathr, feature can be defined by the following characteristics:\n", @@ -360,7 +325,7 @@ "1. Feature source: what source data that this feature is based on\n", "2. Transformation: what transformation is used to transform the source data into feature. Transformation can be optional when you just want to take a column out from the source data.\n", "\n", - "(For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://github.com/linkedin/feathr/blob/main/docs/concepts/feature-definition.md))" + "(For more details on feature definition, please refer to the [Feathr Feature Definition Guide](https://feathr-ai.github.io/feathr/concepts/feature-definition.html))" ] }, { @@ -456,9 +421,7 @@ "source": [ "### Window aggregation features\n", "\n", - "Using [window aggregations](https://en.wikipedia.org/wiki/Window_function_%28SQL%29) can help us create more powerful features. A window aggregation feature compress large amount of information into one single feature value. Using our raw data as an example, we have the users' purchase history data that might be quite some rows, we want to create a window aggregation feature that represents their last 90 days of average purcahse amount.\n", - "\n", - "Feathr provides a nice API to help us create such window aggregation features.\n", + "Using [window aggregations](https://en.wikipedia.org/wiki/Window_function_%28SQL%29) can help us create more powerful features. A window aggregation feature compresses large amount of information into one single feature value. Using our raw data as an example, we have the user's purchase history data that might be quite some rows, we want to create a window aggregation feature that represents their last 90 days of average purchase amount.\n", "\n", "To create this window aggregation feature via Feathr, we just need to define the following parameters with `WindowAggTransformation` API:\n", "1. `agg_expr`: the field/column you want to aggregate. It can be a ANSI SQL expression. So we just write `cast_float(purchase_amount)`(the raw data might be in string form, let's cast_float).\n", @@ -509,9 +472,7 @@ "### Derived Features Section\n", "Derived features are features that are computed from other Feathr features. They could be computed from anchored features, or other derived features.\n", "\n", - "Typical usage includes feature cross(f1 * f2), or computing cosine similarity between two features.\n", - "\n", - "The syntax works in a similar way." + "Typical usage includes feature cross(f1 * f2), or computing cosine similarity between two features. The syntax works in a similar way." ] }, { @@ -524,7 +485,7 @@ " key=user_id,\n", " feature_type=FLOAT,\n", " input_features=[feature_user_gift_card_balance, feature_user_has_valid_credit_card],\n", - " transform=\"feature_user_gift_card_balance + if_else(toBoolean(feature_user_has_valid_credit_card), 100, 0)\")" + " transform=\"feature_user_gift_card_balance + if(boolean(feature_user_has_valid_credit_card), 100, 0)\")" ] }, { @@ -532,7 +493,7 @@ "metadata": {}, "source": [ "### Build Features\n", - "Lastly, we need to build those features so that it can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features." + "Lastly, we need to build these features so that they can be consumed later. Note that we have to build both the \"anchor\" and the \"derived\" features." ] }, { @@ -550,12 +511,11 @@ "metadata": {}, "source": [ "### Optional: A Special Type of Feature: Request Feature\n", - "For advanced user cases, in some cases, features defined on top of request data(a.k.a. observation data) may have no entity key or timestamp.\n", - "It is merely a function/transformation executing against request data at runtime.\n", - "For example, the day of week of the request, which is calculated by converting the request UNIX timestamp.\n", - "In this case, the `source` section should be `INPUT_CONTEXT` to indicate the source of those defined anchors.\n", + "Sometimes features defined on top of request data(a.k.a. observation data) may have no entity key or timestamp. It is merely a function/transformation executing against request data at runtime.\n", + "\n", + "For example, the day of the week of the request, which is calculated by converting the request UNIX timestamp. In this case, the `source` section should be `INPUT_CONTEXT` to indicate the source of those defined anchors.\n", "\n", - "We won't cover the details it in this notebook." + "We won't cover the details of it in this notebook." ] }, { @@ -564,12 +524,11 @@ "source": [ "## Create training data using point-in-time correct feature join\n", "\n", - "A training dataset usually contains entity id column(s), multiple feature columns, event timestamp column and label/target column. \n", + "A training dataset usually contains `entity id` column(s), multiple `feature` columns, event timestamp column and `label/target` column. \n", "\n", - "To create a training dataset using Feathr, we need to provide a feature join settings to specify\n", - "what features and how these features should be joined to the observation data. \n", + "To create a training dataset using Feathr, we need to provide a feature join settings to specify what features and how these features should be joined to the observation data. \n", "\n", - "(To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md))" + "(To learn more on this topic, please refer to [Point-in-time Correctness](https://feathr-ai.github.io/feathr/concepts/point-in-time-join.html))." ] }, { @@ -578,12 +537,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Synapse and Databricks have different output path format\n", - "if feathr_client.spark_runtime == 'databricks':\n", - " output_path = 'dbfs:/feathrazure_test.avro'\n", - "else:\n", - " output_path = feathr_output_path\n", - "\n", + "output_path = feathr_output_path\n", "# Features that we want to request\n", "feature_query = FeatureQuery(feature_list=[\"feature_user_age\", \n", " \"feature_user_tax_rate\", \n", @@ -606,7 +560,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Download the result and show the result\n", + "### Download the result and show the result\n", "\n", "Let's use the helper function `get_result_df` to download the result and view it:" ] @@ -639,7 +593,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Train a machine learning model\n", + "### Train a machine learning model\n", "After getting all the features, let's train a machine learning model with the converted feature by Feathr:" ] }, @@ -694,16 +648,16 @@ "\n", "In the previous section, we demonstrated how Feathr can compute feature value to generate training dataset from feature definition on-they-fly.\n", "\n", - "Now let's talk about how we can use the trained models. We can use the trained models for offline inference as well as online inference. In both cases, we need features to be feed into the models. For offline inference, you can compute and get the features on-demand; or you can store the computed features to some offline database for later offline inference.\n", + "Now let's talk about how we can use the trained models. We can use the trained models for both online and offline inference. In both cases, we need features to be fed into the models. For offline inference, you can compute and get the features on-demand; or you can store the computed features to some offline database for later offline inference.\n", "\n", "For online inference, we can use Feathr to compute and store the features in the online database. Then use it for online inference when the request comes.\n", "\n", - "![img](../images/online_inference.jpg)\n", + "![img](../../images/online_inference.jpg)\n", "\n", "\n", - "In this section, we will focus on materialize features to online store. For materialization to offline store, you can check out our [user guide](https://github.com/linkedin/feathr/blob/main/docs/concepts/materializing-features.md#materializing-features-to-offline-store).\n", + "In this section, we will focus on materialize features to online store. For materialization to offline store, you can check out our [user guide](https://feathr-ai.github.io/feathr/concepts/materializing-features.html#materializing-features-to-offline-store).\n", "\n", - "We can push the computed features to the online store like below:" + "We can push the computed features to the online store(Redis) like below:" ] }, { @@ -721,7 +675,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_user_age\", \"feature_user_gift_card_balance\"])\n", "\n", - "feathr_client.materialize_features(settings)\n", + "feathr_client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "feathr_client.wait_job_to_finish(timeout_sec=500)" ] }, @@ -729,7 +683,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Fetch feature value from online store\n", + "### Fetch feature value from online store\n", "We can then get the features from the online store (Redis) via the client's `get_online_features` or `multi_get_online_features` API." ] }, @@ -761,7 +715,7 @@ "source": [ "### Registering and Fetching features\n", "\n", - "We can also register the features with an Apache Atlas compatible service, such as Azure Purview, and share the registered features across teams:" + "We can also register the features and share them across teams:" ] }, { @@ -771,13 +725,23 @@ "outputs": [], "source": [ "feathr_client.register_features()\n", - "feathr_client.list_registered_features(project_name=\"feathr_getting_started\")" + "feathr_client.list_registered_features(project_name=f\"{FEATHR_PROJECT_NAME}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary\n", + "In this notebook you learnt how to set up Feathr and use it to create features, register features and use those features for model training and inferencing.\n", + "\n", + "We hope this example gave you a good sense of Feathr's capabilities and how you could leverage it within your organization's MLOps workflow." ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.12 ('ifelse_bug_env': venv)", + "display_name": "Python 3.8.13 ('feathrtest')", "language": "python", "name": "python3" }, @@ -791,11 +755,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.12" + "version": "3.8.13" }, "vscode": { "interpreter": { - "hash": "6a6c366ec8f33a88299a9f856c1a3e4312616abcb6fcf46b22c3da0a923e63af" + "hash": "96bbbb728c64ae5eda27ed1c89d74908bf0652fd45caa45cd0ade6bdc0df4d48" } } }, diff --git a/docs/samples/customer360/Customer360.ipynb b/docs/samples/customer360/Customer360.ipynb index 4b202e13a..8d0d8b634 100644 --- a/docs/samples/customer360/Customer360.ipynb +++ b/docs/samples/customer360/Customer360.ipynb @@ -192,9 +192,6 @@ " project_name: 'customer360'\n", " required_environment_variables:\n", " - 'REDIS_PASSWORD'\n", - " - 'AZURE_CLIENT_ID'\n", - " - 'AZURE_TENANT_ID'\n", - " - 'AZURE_CLIENT_SECRET'\n", " - 'ADLS_ACCOUNT'\n", " - 'ADLS_KEY'\n", " - 'WASB_ACCOUNT'\n", @@ -239,10 +236,7 @@ " port: 6380\n", " ssl_enabled: True\n", "feature_registry:\n", - " purview:\n", - " type_system_initialization: true\n", - " purview_name: ''\n", - " delimiter: '__'\n", + " api_endpoint: \"https://.azurewebsites.net/api/v1\"\n", "\"\"\"\n", "# write this configuration string to a temporary location and load it to Feathr\n", "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", @@ -331,9 +325,6 @@ "source": [ "import os\n", "os.environ['REDIS_PASSWORD'] = ''\n", - "os.environ['AZURE_CLIENT_ID'] = ''\n", - "os.environ['AZURE_TENANT_ID'] = ''\n", - "os.environ['AZURE_CLIENT_SECRET'] = ''\n", "os.environ['ADLS_ACCOUNT'] = ''\n", "os.environ['ADLS_KEY'] = ''\n", "os.environ['WASB_ACCOUNT'] = \"\"\n", @@ -863,7 +854,7 @@ " sinks=[redisSink],\n", " feature_names=[\"f_avg_item_ordered_by_customer\",\"f_avg_customer_discount_amount\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=500)" ] }, diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb new file mode 100755 index 000000000..0bc099f11 --- /dev/null +++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_demo.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"843d3142-24ca-4bd1-9e31-b55163804fe3","showTitle":false,"title":""}},"outputs":[],"source":["dbutils.widgets.text(\"RESOURCE_PREFIX\", \"\")\n","dbutils.widgets.text(\"REDIS_KEY\", \"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"384e5e16-7213-4186-9d04-09d03b155534","showTitle":false,"title":""}},"source":["# Feathr Feature Store on Databricks Demo Notebook\n","\n","This notebook illustrates the use of Feature Store to create a model that predicts NYC Taxi fares. The dataset comes from [here](https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page).\n","\n","This notebook is specifically written for Databricks and is relying on some of the Databricks packages such as `dbutils`. The intention here is to provide a \"one click run\" example with minimum configuration. For example:\n","- This notebook skips feature registry which requires running Azure Purview. \n","- To make the online feature query work, you will need to configure the Redis endpoint. \n","\n","The full-fledged notebook can be found from [here](https://github.com/feathr-ai/feathr/blob/main/docs/samples/nyc_taxi_demo.ipynb)."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c2ce58c7-9263-469a-bbb7-43364ddb07b8","showTitle":false,"title":""}},"source":["## Prerequisite\n","\n","To use feathr materialization for online scoring with Redis cache, you may deploy a Redis cluster and set `RESOURCE_PREFIX` and `REDIS_KEY` via Databricks widgets. Note that the deployed Redis host address should be `{RESOURCE_PREFIX}redis.redis.cache.windows.net`. More details about how to deploy the Redis cluster can be found [here](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-cli.html#configurure-redis-cluster).\n","\n","To run this notebook, you'll need to install `feathr` pip package. Here, we install notebook-scoped library. For details, please see [Azure Databricks dependency management document](https://learn.microsoft.com/en-us/azure/databricks/libraries/)."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4609d7ad-ad74-40fc-b97e-f440a0fa0737","showTitle":false,"title":""}},"outputs":[],"source":["!pip install feathr"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c81fa80c-bca6-4ae5-84ad-659a036977bd","showTitle":false,"title":""}},"source":["## Notebook Steps\n","\n","This tutorial demonstrates the key capabilities of Feathr, including:\n","\n","1. Install Feathr and necessary dependencies.\n","1. Create shareable features with Feathr feature definition configs.\n","1. Create training data using point-in-time correct feature join\n","1. Train and evaluate a prediction model.\n","1. Materialize feature values for online scoring.\n","\n","The overall data flow is as follows:\n","\n",""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"80223a02-631c-40c8-91b3-a037249ffff9","showTitle":false,"title":""}},"outputs":[],"source":["from datetime import datetime, timedelta\n","import glob\n","import json\n","from math import sqrt\n","import os\n","from pathlib import Path\n","import requests\n","from tempfile import TemporaryDirectory\n","\n","from azure.identity import AzureCliCredential, DefaultAzureCredential \n","from azure.keyvault.secrets import SecretClient\n","import pandas as pd\n","from pyspark.ml import Pipeline\n","from pyspark.ml.evaluation import RegressionEvaluator\n","from pyspark.ml.feature import VectorAssembler\n","from pyspark.ml.regression import GBTRegressor\n","from pyspark.sql import DataFrame, SparkSession\n","import pyspark.sql.functions as F\n","\n","import feathr\n","from feathr import (\n"," FeathrClient,\n"," # Feature data types\n"," BOOLEAN, FLOAT, INT32, ValueType,\n"," # Feature data sources\n"," INPUT_CONTEXT, HdfsSource,\n"," # Feature aggregations\n"," TypedKey, WindowAggTransformation,\n"," # Feature types and anchor\n"," DerivedFeature, Feature, FeatureAnchor,\n"," # Materialization\n"," BackfillTime, MaterializationSettings, RedisSink,\n"," # Offline feature computation\n"," FeatureQuery, ObservationSettings,\n",")\n","from feathr.datasets import nyc_taxi\n","from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration\n","from feathr.utils.config import generate_config\n","from feathr.utils.job_utils import get_result_df\n","\n","\n","print(f\"\"\"Feathr version: {feathr.__version__}\n","Databricks runtime version: {spark.conf.get(\"spark.databricks.clusterUsageTags.sparkVersion\")}\"\"\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ab35fa01-b392-457e-8fde-7e445a3c39b5","showTitle":false,"title":""}},"source":["## 2. Create Shareable Features with Feathr Feature Definition Configs\n","\n","In this notebook, we define all the necessary resource key values for authentication. We use the values passed by the databricks widgets at the top of this notebook. Instead of manually entering the values to the widgets, we can also use [Azure Key Vault](https://azure.microsoft.com/en-us/services/key-vault/) to retrieve them.\n","Please refer to [how-to guide documents for granting key-vault access](https://feathr-ai.github.io/feathr/how-to-guides/azure-deployment-arm.html#3-grant-key-vault-and-synapse-access-to-selected-users-optional) and [Databricks' Azure Key Vault-backed scopes](https://learn.microsoft.com/en-us/azure/databricks/security/secrets/secret-scopes) for more details."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"09f93a9f-7b33-4d91-8f31-ee3b20991696","showTitle":false,"title":""}},"outputs":[],"source":["RESOURCE_PREFIX = dbutils.widgets.get(\"RESOURCE_PREFIX\")\n","PROJECT_NAME = \"feathr_getting_started\"\n","\n","REDIS_KEY = dbutils.widgets.get(\"REDIS_KEY\")\n","\n","# Use a databricks cluster\n","SPARK_CLUSTER = \"databricks\"\n","\n","# Databricks file system path\n","DATA_STORE_PATH = f\"dbfs:/{PROJECT_NAME}\""]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"41d3648a-9bc9-40dc-90da-bc82b21ef9b3","showTitle":false,"title":""}},"source":["In the following cell, we set required databricks credentials automatically by using a databricks notebook context object as well as new job cluster spec.\n","\n","Note: When submitting jobs, Databricks recommend to use new clusters for greater reliability. If you want to use an existing all-purpose cluster, you may set\n","`existing_cluster_id': ctx.tags().get('clusterId').get()` to the `databricks_config`, replacing `new_cluster` config values."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"331753d6-1850-47b5-ad97-84b7c01d79d1","showTitle":false,"title":""}},"outputs":[],"source":["# Redis credential\n","os.environ['REDIS_PASSWORD'] = REDIS_KEY\n","\n","# Setup databricks env configs\n","ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext()\n","databricks_config = {\n"," 'run_name': \"FEATHR_FILL_IN\",\n"," # To use an existing all-purpose cluster:\n"," # 'existing_cluster_id': ctx.tags().get('clusterId').get(),\n"," # To use a new job cluster:\n"," 'new_cluster': {\n"," 'spark_version': \"11.2.x-scala2.12\",\n"," 'node_type_id': \"Standard_D3_v2\",\n"," 'num_workers':1,\n"," 'spark_conf': {\n"," 'FEATHR_FILL_IN': \"FEATHR_FILL_IN\",\n"," # Exclude conflicting packages if use feathr <= v0.8.0:\n"," 'spark.jars.excludes': \"commons-logging:commons-logging,org.slf4j:slf4j-api,com.google.protobuf:protobuf-java,javax.xml.bind:jaxb-api\",\n"," },\n"," },\n"," 'libraries': [{'jar': \"FEATHR_FILL_IN\"}],\n"," 'spark_jar_task': {\n"," 'main_class_name': \"FEATHR_FILL_IN\",\n"," 'parameters': [\"FEATHR_FILL_IN\"],\n"," },\n","}\n","os.environ['spark_config__databricks__workspace_instance_url'] = \"https://\" + ctx.tags().get('browserHostName').get()\n","os.environ['spark_config__databricks__config_template'] = json.dumps(databricks_config)\n","os.environ['spark_config__databricks__work_dir'] = \"dbfs:/feathr_getting_started\"\n","os.environ['DATABRICKS_WORKSPACE_TOKEN_VALUE'] = ctx.apiToken().get()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"08bc3b7e-bbf5-4e3a-9978-fe1aef8c1aee","showTitle":false,"title":""}},"source":["### Configurations\n","\n","Feathr uses a yaml file to define configurations. Please refer to [feathr_config.yaml]( https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for the meaning of each field."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8cd64e3a-376c-48e6-ba41-5197f3591d48","showTitle":false,"title":""}},"outputs":[],"source":["config_path = generate_config(project_name=PROJECT_NAME, spark_cluster=SPARK_CLUSTER, resource_prefix=RESOURCE_PREFIX)\n","\n","with open(config_path, 'r') as f: \n"," print(f.read())"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"58d22dc1-7590-494d-94ca-3e2488c31c8e","showTitle":false,"title":""}},"source":["All the configurations can be overwritten by environment variables with concatenation of `__` for different layers of the config file. For example, `feathr_runtime_location` for databricks config can be overwritten by setting `spark_config__databricks__feathr_runtime_location` environment variable."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3fef7f2f-df19-4f53-90a5-ff7999ed983d","showTitle":false,"title":""}},"source":["### Initialize Feathr Client"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9713a2df-c7b2-4562-88b0-b7acce3cc43a","showTitle":false,"title":""}},"outputs":[],"source":["client = FeathrClient(config_path=config_path)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c3b64bda-d42c-4a64-b976-0fb604cf38c5","showTitle":false,"title":""}},"source":["### View the NYC taxi fare dataset"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"c4ccd7b3-298a-4e5a-8eec-b7e309db393e","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FILE_PATH = str(Path(DATA_STORE_PATH, \"nyc_taxi.csv\"))\n","\n","# Download the data file\n","df_raw = nyc_taxi.get_spark_df(spark=spark, local_cache_path=DATA_FILE_PATH)\n","df_raw.limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"7430c942-64e5-4b70-b823-16ce1d1b3cee","showTitle":false,"title":""}},"source":["### Defining features with Feathr\n","\n","In Feathr, a feature is viewed as a function, mapping a key and timestamp to a feature value. For more details, please see [Feathr Feature Definition Guide](https://github.com/feathr-ai/feathr/blob/main/docs/concepts/feature-definition.md).\n","\n","* The feature key (a.k.a. entity id) identifies the subject of feature, e.g. a user_id or location_id.\n","* The feature name is the aspect of the entity that the feature is indicating, e.g. the age of the user.\n","* The feature value is the actual value of that aspect at a particular time, e.g. the value is 30 at year 2022.\n","\n","Note that, in some cases, a feature could be just a transformation function that has no entity key or timestamp involved, e.g. *the day of week of the request timestamp*.\n","\n","There are two types of features -- anchored features and derivated features:\n","\n","* **Anchored features**: Features that are directly extracted from sources. Could be with or without aggregation. \n","* **Derived features**: Features that are computed on top of other features.\n","\n","#### Define anchored features\n","\n","A feature source is needed for anchored features that describes the raw data in which the feature values are computed from. A source value should be either `INPUT_CONTEXT` (the features that will be extracted from the observation data directly) or `feathr.source.Source` object."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"75b8d2ed-84df-4446-ae07-5f715434f3ea","showTitle":false,"title":""}},"outputs":[],"source":["TIMESTAMP_COL = \"lpep_dropoff_datetime\"\n","TIMESTAMP_FORMAT = \"yyyy-MM-dd HH:mm:ss\""]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"93abbcc2-562b-47e4-ad4c-1fedd7cc64df","showTitle":false,"title":""}},"outputs":[],"source":["# We define f_trip_distance and f_trip_time_duration features separately\n","# so that we can reuse them later for the derived features.\n","f_trip_distance = Feature(\n"," name=\"f_trip_distance\",\n"," feature_type=FLOAT,\n"," transform=\"trip_distance\",\n",")\n","f_trip_time_duration = Feature(\n"," name=\"f_trip_time_duration\",\n"," feature_type=FLOAT,\n"," transform=\"cast_float((to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime)) / 60)\",\n",")\n","\n","features = [\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," Feature(\n"," name=\"f_is_long_trip_distance\",\n"," feature_type=BOOLEAN,\n"," transform=\"trip_distance > 30.0\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_week\",\n"," feature_type=INT32,\n"," transform=\"dayofweek(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_day_of_month\",\n"," feature_type=INT32,\n"," transform=\"dayofmonth(lpep_dropoff_datetime)\",\n"," ),\n"," Feature(\n"," name=\"f_hour_of_day\",\n"," feature_type=INT32,\n"," transform=\"hour(lpep_dropoff_datetime)\",\n"," ),\n","]\n","\n","# After you have defined features, bring them together to build the anchor to the source.\n","feature_anchor = FeatureAnchor(\n"," name=\"feature_anchor\",\n"," source=INPUT_CONTEXT, # Pass through source, i.e. observation data.\n"," features=features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"728d2d5f-c11f-4941-bdc5-48507f5749f1","showTitle":false,"title":""}},"source":["We can define the source with a preprocessing python function."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3cc59a0e-a41b-480e-a84e-ca5443d63143","showTitle":false,"title":""}},"outputs":[],"source":["def preprocessing(df: DataFrame) -> DataFrame:\n"," import pyspark.sql.functions as F\n"," df = df.withColumn(\"fare_amount_cents\", (F.col(\"fare_amount\") * 100.0).cast(\"float\"))\n"," return df\n","\n","batch_source = HdfsSource(\n"," name=\"nycTaxiBatchSource\",\n"," path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," preprocessing=preprocessing,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"46f863c4-bb81-434a-a448-6b585031a221","showTitle":false,"title":""}},"source":["For the features with aggregation, the supported functions are as follows:\n","\n","| Aggregation Function | Input Type | Description |\n","| --- | --- | --- |\n","|SUM, COUNT, MAX, MIN, AVG\t|Numeric|Applies the the numerical operation on the numeric inputs. |\n","|MAX_POOLING, MIN_POOLING, AVG_POOLING\t| Numeric Vector | Applies the max/min/avg operation on a per entry bassis for a given a collection of numbers.|\n","|LATEST| Any |Returns the latest not-null values from within the defined time window |"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"a373ecbe-a040-4cd3-9d87-0d5f4c5ba553","showTitle":false,"title":""}},"outputs":[],"source":["agg_key = TypedKey(\n"," key_column=\"DOLocationID\",\n"," key_column_type=ValueType.INT32,\n"," description=\"location id in NYC\",\n"," full_name=\"nyc_taxi.location_id\",\n",")\n","\n","agg_window = \"90d\"\n","\n","# Anchored features with aggregations\n","agg_features = [\n"," Feature(\n"," name=\"f_location_avg_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"AVG\",\n"," window=agg_window,\n"," ),\n"," ),\n"," Feature(\n"," name=\"f_location_max_fare\",\n"," key=agg_key,\n"," feature_type=FLOAT,\n"," transform=WindowAggTransformation(\n"," agg_expr=\"fare_amount_cents\",\n"," agg_func=\"MAX\",\n"," window=agg_window,\n"," ),\n"," ),\n","]\n","\n","agg_feature_anchor = FeatureAnchor(\n"," name=\"agg_feature_anchor\",\n"," source=batch_source, # External data source for feature. Typically a data table.\n"," features=agg_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"149f85e2-fa3c-4895-b0c5-de5543ca9b6d","showTitle":false,"title":""}},"source":["#### Define derived features\n","\n","We also define a derived feature, `f_trip_time_distance`, from the anchored features `f_trip_distance` and `f_trip_time_duration` as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"05633bc3-9118-449b-9562-45fc437576c2","showTitle":false,"title":""}},"outputs":[],"source":["derived_features = [\n"," DerivedFeature(\n"," name=\"f_trip_time_distance\",\n"," feature_type=FLOAT,\n"," input_features=[\n"," f_trip_distance,\n"," f_trip_time_duration,\n"," ],\n"," transform=\"f_trip_distance / f_trip_time_duration\",\n"," )\n","]"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"ad102c45-586d-468c-85f0-9454401ef10b","showTitle":false,"title":""}},"source":["### Build features\n","\n","Finally, we build the features."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"91bb5ebb-87e4-470b-b8eb-1c89b351740e","showTitle":false,"title":""}},"outputs":[],"source":["client.build_features(\n"," anchor_list=[feature_anchor, agg_feature_anchor],\n"," derived_feature_list=derived_features,\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"632d5f46-f9e2-41a8-aab7-34f75206e2aa","showTitle":false,"title":""}},"source":["## 3. Create Training Data Using Point-in-Time Correct Feature Join\n","\n","After the feature producers have defined the features (as described in the Feature Definition part), the feature consumers may want to consume those features. Feature consumers will use observation data to query from different feature tables using Feature Query.\n","\n","To create a training dataset using Feathr, one needs to provide a feature join configuration file to specify\n","what features and how these features should be joined to the observation data. \n","\n","To learn more on this topic, please refer to [Point-in-time Correctness](https://github.com/linkedin/feathr/blob/main/docs/concepts/point-in-time-join.md)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"02feabc9-2f2f-43e8-898d-b28082798e98","showTitle":false,"title":""}},"outputs":[],"source":["feature_names = [feature.name for feature in features + agg_features + derived_features]\n","feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"e438e6d8-162e-4aa3-b3b3-9d1f3b0d2b7f","showTitle":false,"title":""}},"outputs":[],"source":["DATA_FORMAT = \"parquet\"\n","offline_features_path = str(Path(DATA_STORE_PATH, \"feathr_output\", f\"features.{DATA_FORMAT}\"))"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"67e81466-c736-47ba-b122-e640642c01cf","showTitle":false,"title":""}},"outputs":[],"source":["# Features that we want to request. Can use a subset of features\n","query = FeatureQuery(\n"," feature_list=feature_names,\n"," key=agg_key,\n",")\n","settings = ObservationSettings(\n"," observation_path=DATA_FILE_PATH,\n"," event_timestamp_column=TIMESTAMP_COL,\n"," timestamp_format=TIMESTAMP_FORMAT,\n",")\n","client.get_offline_features(\n"," observation_settings=settings,\n"," feature_query=query,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," # For more details, see https://feathr-ai.github.io/feathr/how-to-guides/feathr-job-configuration.html\n"," execution_configurations=SparkExecutionConfiguration({\n"," \"spark.feathr.outputFormat\": DATA_FORMAT,\n"," }),\n"," output_path=offline_features_path,\n",")\n","\n","client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"9871af55-25eb-41ee-a58a-fda74b1a174e","showTitle":false,"title":""}},"outputs":[],"source":["# Show feature results\n","df = get_result_df(\n"," spark=spark,\n"," client=client,\n"," data_format=\"parquet\",\n"," res_url=offline_features_path,\n",")\n","df.select(feature_names).limit(5).toPandas()"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"dcbf17fc-7f79-4a65-a3af-9cffbd0b5d1f","showTitle":false,"title":""}},"source":["## 4. Train and Evaluate a Prediction Model\n","\n","After generating all the features, we train and evaluate a machine learning model to predict the NYC taxi fare prediction. In this example, we use Spark MLlib's [GBTRegressor](https://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-tree-regression).\n","\n","Note that designing features, training prediction models and evaluating them are an iterative process where the models' performance maybe used to modify the features as a part of the modeling process."]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5a226026-1c7b-48db-8f91-88d5c2ddf023","showTitle":false,"title":""}},"source":["### Load Train and Test Data from the Offline Feature Values"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bd2cdc83-0920-46e8-9454-e5e6e7832ce0","showTitle":false,"title":""}},"outputs":[],"source":["# Train / test split\n","train_df, test_df = (\n"," df # Dataframe that we generated from get_offline_features call.\n"," .withColumn(\"label\", F.col(\"fare_amount\").cast(\"double\"))\n"," .where(F.col(\"f_trip_time_duration\") > 0)\n"," .fillna(0)\n"," .randomSplit([0.8, 0.2])\n",")\n","\n","print(f\"Num train samples: {train_df.count()}\")\n","print(f\"Num test samples: {test_df.count()}\")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"6a3e2ab1-5c66-4d27-a737-c5e2af03b1dd","showTitle":false,"title":""}},"source":["### Build a ML Pipeline\n","\n","Here, we use Spark ML Pipeline to aggregate feature vectors and feed them to the model."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"2a254361-63e9-45b2-8c19-40549762eacb","showTitle":false,"title":""}},"outputs":[],"source":["# Generate a feature vector column for SparkML\n","vector_assembler = VectorAssembler(\n"," inputCols=[x for x in df.columns if x in feature_names],\n"," outputCol=\"features\",\n",")\n","\n","# Define a model\n","gbt = GBTRegressor(\n"," featuresCol=\"features\",\n"," maxIter=100,\n"," maxDepth=5,\n"," maxBins=16,\n",")\n","\n","# Create a ML pipeline\n","ml_pipeline = Pipeline(stages=[\n"," vector_assembler,\n"," gbt,\n","])"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"bef93538-9591-4247-97b6-289d2055b7b1","showTitle":false,"title":""}},"source":["### Train and Evaluate the Model"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"0c3d5f35-11a3-4644-9992-5860169d8302","showTitle":false,"title":""}},"outputs":[],"source":["# Train a model\n","model = ml_pipeline.fit(train_df)\n","\n","# Make predictions\n","predictions = model.transform(test_df)"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"1f9b584c-6228-4a02-a6c3-9b8dd2b78091","showTitle":false,"title":""}},"outputs":[],"source":["# Evaluate\n","evaluator = RegressionEvaluator(\n"," labelCol=\"label\",\n"," predictionCol=\"prediction\",\n",")\n","\n","rmse = evaluator.evaluate(predictions, {evaluator.metricName: \"rmse\"})\n","mae = evaluator.evaluate(predictions, {evaluator.metricName: \"mae\"})\n","print(f\"RMSE: {rmse}\\nMAE: {mae}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"25c33abd-6e87-437d-a6a1-86435f065a1e","showTitle":false,"title":""}},"outputs":[],"source":["# predicted fare vs actual fare plots -- will this work for databricks / synapse / local ?\n","predictions_pdf = predictions.select([\"label\", \"prediction\"]).toPandas().reset_index()\n","\n","predictions_pdf.plot(\n"," x=\"index\",\n"," y=[\"label\", \"prediction\"],\n"," style=['-', ':'],\n"," figsize=(20, 10),\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"664d78cc-4a92-430c-9e05-565ba904558e","showTitle":false,"title":""}},"outputs":[],"source":["predictions_pdf.plot.scatter(\n"," x=\"label\",\n"," y=\"prediction\",\n"," xlim=(0, 100),\n"," ylim=(0, 100),\n"," figsize=(10, 10),\n",")"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"8a56d165-c813-4ce0-8ae6-9f4d313c463d","showTitle":false,"title":""}},"source":["## 5. Materialize Feature Values for Online Scoring\n","\n","While we computed feature values on-the-fly at request time via Feathr, we can pre-compute the feature values and materialize them to offline or online storages such as Redis.\n","\n","Note, only the features anchored to offline data source can be materialized."]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"751fa72e-8f94-40a1-994e-3e8315b51d37","showTitle":false,"title":""}},"outputs":[],"source":["materialized_feature_names = [feature.name for feature in agg_features]\n","materialized_feature_names"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"4d4699ed-42e6-408f-903d-2f799284f4b6","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," FEATURE_TABLE_NAME = \"nycTaxiDemoFeature\"\n","\n"," # Get the last date from the dataset\n"," backfill_timestamp = (\n"," df_raw\n"," .select(F.to_timestamp(F.col(TIMESTAMP_COL), TIMESTAMP_FORMAT).alias(TIMESTAMP_COL))\n"," .agg({TIMESTAMP_COL: \"max\"})\n"," .collect()[0][0]\n"," )\n","\n"," # Time range to materialize\n"," backfill_time = BackfillTime(\n"," start=backfill_timestamp,\n"," end=backfill_timestamp,\n"," step=timedelta(days=1),\n"," )\n","\n"," # Destinations:\n"," # For online store,\n"," redis_sink = RedisSink(table_name=FEATURE_TABLE_NAME)\n","\n"," # For offline store,\n"," # adls_sink = HdfsSink(output_path=)\n","\n"," settings = MaterializationSettings(\n"," name=FEATURE_TABLE_NAME + \".job\", # job name\n"," backfill_time=backfill_time,\n"," sinks=[redis_sink], # or adls_sink\n"," feature_names=materialized_feature_names,\n"," )\n","\n"," client.materialize_features(\n"," settings=settings,\n"," # Note, execution_configurations argument only works when using a new job cluster\n"," execution_configurations={\"spark.feathr.outputFormat\": \"parquet\"},\n"," )\n","\n"," client.wait_job_to_finish(timeout_sec=500)"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"5aa13acd-58ec-4fc2-86bb-dc1d9951ebb9","showTitle":false,"title":""}},"source":["Now, you can retrieve features for online scoring as follows:"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"424bc9eb-a47f-4b46-be69-8218d55e66ad","showTitle":false,"title":""}},"outputs":[],"source":["if REDIS_KEY and RESOURCE_PREFIX:\n"," # Note, to get a single key, you may use client.get_online_features instead\n"," materialized_feature_values = client.multi_get_online_features(\n"," feature_table=FEATURE_TABLE_NAME,\n"," keys=[\"239\", \"265\"],\n"," feature_names=materialized_feature_names,\n"," )\n"," materialized_feature_values"]},{"cell_type":"markdown","metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"3596dc71-a363-4b6a-a169-215c89978558","showTitle":false,"title":""}},"source":["## Cleanup"]},{"cell_type":"code","execution_count":null,"metadata":{"application/vnd.databricks.v1+cell":{"inputWidgets":{},"nuid":"b5fb292e-bbb6-4dd7-8e79-c62d9533e820","showTitle":false,"title":""}},"outputs":[],"source":["# Remove temporary files\n","dbutils.fs.rm(\"dbfs:/tmp/\", recurse=True)"]}],"metadata":{"application/vnd.databricks.v1+notebook":{"dashboards":[],"language":"python","notebookMetadata":{"pythonIndentUnit":4},"notebookName":"databricks_quickstart_nyc_taxi_demo","notebookOrigID":2365994027381987,"widgets":{"REDIS_KEY":{"currentValue":"","nuid":"d39ce0d5-bcfe-47ef-b3d9-eff67e5cdeca","widgetInfo":{"defaultValue":"","label":null,"name":"REDIS_KEY","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}},"RESOURCE_PREFIX":{"currentValue":"","nuid":"87a26035-86fc-4dbd-8dd0-dc546c1c63c1","widgetInfo":{"defaultValue":"","label":null,"name":"RESOURCE_PREFIX","options":{"validationRegex":null,"widgetType":"text"},"widgetType":"text"}}}},"kernelspec":{"display_name":"Python 3.10.8 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.8"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}}},"nbformat":4,"nbformat_minor":0} diff --git a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb index 82aaf3832..32a880431 100644 --- a/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb +++ b/docs/samples/databricks/databricks_quickstart_nyc_taxi_driver.ipynb @@ -243,7 +243,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "You need to setup the Redis credentials below in order to push features to online store. You can skip this part if you don't have Redis, but there will be failures for `client.materialize_features(settings)` API." + "You need to setup the Redis credentials below in order to push features to online store. You can skip this part if you don't have Redis, but there will be failures for `client.materialize_features(settings, allow_materialize_non_agg_feature =True)` API." ] }, { @@ -332,9 +332,6 @@ " project_name: 'feathr_getting_started2'\n", " required_environment_variables:\n", " - 'REDIS_PASSWORD'\n", - " - 'AZURE_CLIENT_ID'\n", - " - 'AZURE_TENANT_ID'\n", - " - 'AZURE_CLIENT_SECRET'\n", "offline_store:\n", " adls:\n", " adls_enabled: true\n", @@ -364,10 +361,7 @@ " port: 6380\n", " ssl_enabled: True\n", "feature_registry:\n", - " purview:\n", - " type_system_initialization: true\n", - " purview_name: ''\n", - " delimiter: '__'\n", + " api_endpoint: \"https://.azurewebsites.net/api/v1\"\n", "\"\"\"\n", "tmp = tempfile.NamedTemporaryFile(mode='w', delete=False)\n", "with open(tmp.name, \"w\") as text_file:\n", @@ -1277,7 +1271,7 @@ " sinks=[redisSink],\n", " feature_names=[\"f_location_avg_fare\", \"f_location_max_fare\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=500)\n" ] }, @@ -1422,11 +1416,8 @@ "notebookOrigID": 930353059183053, "widgets": {} }, - "interpreter": { - "hash": "830c16c5b424e7ff512f67d4056b67cea1a756a7ad6a92c98b9e2b95c5e484ae" - }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.8.10 ('logistics')", "language": "python", "name": "python3" }, @@ -1440,7 +1431,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "6d25d3d1f1809ed0384c3d8e0cd4f1df57fe7bb936ead67f035c6ff1494f4e23" + } } }, "nbformat": 4, diff --git a/docs/samples/fraud_detection_demo.ipynb b/docs/samples/fraud_detection_demo.ipynb index 88c672160..2b5da39d3 100644 --- a/docs/samples/fraud_detection_demo.ipynb +++ b/docs/samples/fraud_detection_demo.ipynb @@ -169,7 +169,7 @@ "adls_fs_name=resource_prefix+\"fs\"\n", "purview_name=resource_prefix+\"purview\"\n", "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", + "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False, additionally_allowed_tenants=['*'])\n", "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", "retrieved_secret = client.get_secret(secretName).value\n", @@ -284,7 +284,7 @@ }, "outputs": [], "source": [ - "client = FeathrClient(config_path=tmp.name)" + "client = FeathrClient(config_path=tmp.name, credential=credential)" ] }, { @@ -899,7 +899,7 @@ " sinks=[redisSink],\n", " feature_names=[\"fraud_status\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=5000)" ] }, diff --git a/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb b/docs/samples/nyc_taxi_demo.ipynb similarity index 98% rename from feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb rename to docs/samples/nyc_taxi_demo.ipynb index 38cec2ca9..0de7662b2 100644 --- a/feathr_project/feathrcli/data/feathr_user_workspace/nyc_driver_demo.ipynb +++ b/docs/samples/nyc_taxi_demo.ipynb @@ -147,7 +147,7 @@ "adls_fs_name=resource_prefix+\"fs\"\n", "purview_name=resource_prefix+\"purview\"\n", "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", + "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False, additionally_allowed_tenants=['*'])\n", "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", "retrieved_secret = client.get_secret(secretName).value\n", @@ -630,7 +630,7 @@ " sinks=[redisSink],\n", " feature_names=[\"f_location_avg_fare\", \"f_location_max_fare\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=500)\n" ] }, @@ -693,7 +693,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9.5 ('base')", + "display_name": "Python 3.10.8 64-bit", "language": "python", "name": "python3" }, @@ -707,11 +707,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.10.8" }, "vscode": { "interpreter": { - "hash": "3d597f4c481aa0f25dceb95d2a0067e73c0966dcbd003d741d821a7208527ecf" + "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e" } } }, diff --git a/docs/samples/product_recommendation_demo_advanced.ipynb b/docs/samples/product_recommendation_demo_advanced.ipynb index fff2a1cd5..ecaff5852 100644 --- a/docs/samples/product_recommendation_demo_advanced.ipynb +++ b/docs/samples/product_recommendation_demo_advanced.ipynb @@ -20,7 +20,7 @@ "\n", "After the model is trained, given a user_id, product_id pair and features, we should be able to predict the product rating that the user will give for this product_id.\n", "\n", - "(Compared with [the beginner version of product recommendation](product_recommendation_demo.ipynb), this tutorial expanded the example by predicting ratings for all products.)\n", + "(Compared with [the beginner version of product recommendation](https://github.com/feathr-ai/feathr/blob/main/docs/samples/azure_synapse/product_recommendation_demo.ipynb), this tutorial expanded the example by predicting ratings for all products.)\n", "\n", "## Feature Creation Illustration\n", "In this example, our observation data has compound entity key where a record is uniquely identified by user_id and product_id. So there might be 3 types of features:\n", @@ -116,7 +116,7 @@ }, "outputs": [], "source": [ - "resource_prefix = \"ckim2\"" + "resource_prefix = \"feathr_resource_prefix\"" ] }, { @@ -270,7 +270,7 @@ "adls_fs_name=resource_prefix+\"fs\"\n", "purview_name=resource_prefix+\"purview\"\n", "key_vault_uri = f\"https://{key_vault_name}.vault.azure.net\"\n", - "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False)\n", + "credential = DefaultAzureCredential(exclude_interactive_browser_credential=False, additionally_allowed_tenants=['*'])\n", "client = SecretClient(vault_url=key_vault_uri, credential=credential)\n", "secretName = \"FEATHR-ONLINE-STORE-CONN\"\n", "retrieved_secret = client.get_secret(secretName).value\n", @@ -389,7 +389,7 @@ "\n", "You should setup the environment variables in order to run this sample. More environment variables can be set by referring to [feathr_config.yaml](https://github.com/linkedin/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) and use that as the source of truth. It also has more explanations on the meaning of each variable.\n", "\n", - "To run this notebook, for Azure users, you need AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET and REDIS_PASSWORD.\n", + "To run this notebook, for Azure users, you need REDIS_PASSWORD.\n", "To run this notebook, for Databricks useres, you need DATABRICKS_WORKSPACE_TOKEN_VALUE and REDIS_PASSWORD." ] }, @@ -420,7 +420,7 @@ }, "outputs": [], "source": [ - "client = FeathrClient(config_path=tmp.name)" + "client = FeathrClient(config_path=tmp.name, credential=credential)" ] }, { @@ -781,7 +781,7 @@ " feature_type=FLOAT,\n", " input_features=[\n", " feature_user_gift_card_balance, feature_user_has_valid_credit_card],\n", - " transform=\"feature_user_gift_card_balance + if_else(toBoolean(feature_user_has_valid_credit_card), 100, 0)\")" + " transform=\"feature_user_gift_card_balance + if(boolean(feature_user_has_valid_credit_card), 100, 0)\")" ] }, { @@ -1037,7 +1037,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_user_age\", \"feature_user_gift_card_balance\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=1000)" ] }, @@ -1144,7 +1144,7 @@ " sinks=[redisSink],\n", " feature_names=[\"feature_product_price\"])\n", "\n", - "client.materialize_features(settings)\n", + "client.materialize_features(settings, allow_materialize_non_agg_feature =True)\n", "client.wait_job_to_finish(timeout_sec=1000)" ] }, @@ -1214,7 +1214,7 @@ "widgets": {} }, "kernelspec": { - "display_name": "Python 3.9.5 ('base')", + "display_name": "Python 3.9.13 64-bit ('3.9.13')", "language": "python", "name": "python3" }, @@ -1228,11 +1228,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.9.13" }, "vscode": { "interpreter": { - "hash": "3d597f4c481aa0f25dceb95d2a0067e73c0966dcbd003d741d821a7208527ecf" + "hash": "c5d1b88564ea095927319e95d120a01ba9530a1c584720276480e541fd6461c7" } } }, diff --git a/feathr_project/feathr/__init__.py b/feathr_project/feathr/__init__.py index 695856883..5c279b7d5 100644 --- a/feathr_project/feathr/__init__.py +++ b/feathr_project/feathr/__init__.py @@ -17,6 +17,7 @@ from .definition.settings import * from .utils.job_utils import * from .utils.feature_printer import * +from .version import __version__ # skipped class as they are internal methods: # RepoDefinitions, HoconConvertible, @@ -74,4 +75,5 @@ 'ObservationSettings', 'FeaturePrinter', 'SparkExecutionConfiguration', + __version__, ] diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index 4c7a97e6a..f1c692c4e 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -1,39 +1,37 @@ import base64 +import copy import logging import os import tempfile from typing import Dict, List, Union -from feathr.definition.feature import FeatureBase -import copy -import redis from azure.identity import DefaultAzureCredential +from feathr.definition.transformation import WindowAggTransformation from jinja2 import Template from pyhocon import ConfigFactory -from feathr.definition.sink import Sink -from feathr.registry.feature_registry import default_registry_client - -from feathr.spark_provider._databricks_submission import _FeathrDatabricksJobLauncher -from feathr.spark_provider._synapse_submission import _FeathrSynapseJobLauncher -from feathr.spark_provider._localspark_submission import _FeathrDLocalSparkJobLauncher +import redis -from feathr.definition._materialization_utils import _to_materialization_config -from feathr.udf._preprocessing_pyudf_manager import _PreprocessingPyudfManager from feathr.constants import * -from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration +from feathr.definition._materialization_utils import _to_materialization_config +from feathr.definition.anchor import FeatureAnchor +from feathr.definition.feature import FeatureBase from feathr.definition.feature_derivations import DerivedFeature from feathr.definition.materialization_settings import MaterializationSettings from feathr.definition.monitoring_settings import MonitoringSettings -from feathr.protobuf.featureValue_pb2 import FeatureValue from feathr.definition.query_feature_list import FeatureQuery from feathr.definition.settings import ObservationSettings -from feathr.definition.feature_derivations import DerivedFeature -from feathr.definition.anchor import FeatureAnchor +from feathr.definition.sink import Sink +from feathr.protobuf.featureValue_pb2 import FeatureValue +from feathr.registry.feature_registry import default_registry_client +from feathr.spark_provider._databricks_submission import _FeathrDatabricksJobLauncher +from feathr.spark_provider._localspark_submission import _FeathrLocalSparkJobLauncher +from feathr.spark_provider._synapse_submission import _FeathrSynapseJobLauncher from feathr.spark_provider.feathr_configurations import SparkExecutionConfiguration +from feathr.udf._preprocessing_pyudf_manager import _PreprocessingPyudfManager from feathr.utils._envvariableutil import _EnvVaraibleUtil from feathr.utils._file_utils import write_to_file from feathr.utils.feature_printer import FeaturePrinter -from feathr.utils.spark_job_params import FeatureJoinJobParams, FeatureGenerationJobParams +from feathr.utils.spark_job_params import FeatureGenerationJobParams, FeatureJoinJobParams class FeathrClient(object): @@ -111,7 +109,7 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir self.credential = credential if self.spark_runtime not in {'azure_synapse', 'databricks', 'local'}: raise RuntimeError( - 'Only \'azure_synapse\' and \'databricks\' are currently supported.') + f'{self.spark_runtime} is not supported. Only \'azure_synapse\', \'databricks\' and \'local\' are currently supported.') elif self.spark_runtime == 'azure_synapse': # Feathr is a spark-based application so the feathr jar compiled from source code will be used in the # Spark job submission. The feathr jar hosted in cloud saves the time users needed to upload the jar from @@ -158,7 +156,7 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir self._FEATHR_JOB_JAR_PATH = \ self.envutils.get_environment_variable_with_default( 'spark_config', 'local', 'feathr_runtime_location') - self.feathr_spark_launcher = _FeathrDLocalSparkJobLauncher( + self.feathr_spark_launcher = _FeathrLocalSparkJobLauncher( workspace_path = self.envutils.get_environment_variable_with_default('spark_config', 'local', 'workspace'), master = self.envutils.get_environment_variable_with_default('spark_config', 'local', 'master') ) @@ -202,7 +200,7 @@ def build_features(self, anchor_list: List[FeatureAnchor] = [], derived_feature_ f"definitions. Anchor name of {anchor} is already defined in {anchor_names[anchor.name]}") else: anchor_names[anchor.name] = anchor - if anchor.source.name in source_names: + if anchor.source.name in source_names and (anchor.source is not source_names[anchor.source.name]): raise RuntimeError(f"Source name should be unique but there are duplicate source names in your source " f"definitions. Source name of {anchor.source} is already defined in {source_names[anchor.source.name]}") else: @@ -354,7 +352,7 @@ def _decode_proto(self, feature_list): else: typed_result.append(raw_feature) return typed_result - + def delete_feature_from_redis(self, feature_table, key, feature_name) -> None: """ Delete feature from Redis @@ -364,7 +362,7 @@ def delete_feature_from_redis(self, feature_table, key, feature_name) -> None: key: the key of the entity feature_name: feature name to be deleted """ - + redis_key = self._construct_redis_key(feature_table, key) if self.redis_client.hexists(redis_key, feature_name): self.redis_client.delete(redis_key, feature_name) @@ -574,20 +572,20 @@ def monitor_features(self, settings: MonitoringSettings, execution_configuration def _get_feature_key(self, feature_name: str): features: List[FeatureBase] = [] if 'derived_feature_list' in dir(self): - features += self.derived_feature_list + features += self.derived_feature_list if 'anchor_list' in dir(self): for anchor in self.anchor_list: - features += anchor.features + features += anchor.features for feature in features: if feature.name == feature_name: keys = feature.key - return set(key.key_column for key in keys) + return set(key.key_column for key in keys) self.logger.warning(f"Invalid feature name: {feature_name}. Please call FeathrClient.build_features() first in order to materialize the features.") return None - + # Validation on feature keys: # Features within a set of aggregation or planned to be merged should have same keys - # The param "allow_empty_key" shows if empty keys are acceptable + # The param "allow_empty_key" shows if empty keys are acceptable def _valid_materialize_keys(self, features: List[str], allow_empty_key=False): keys = None for feature in features: @@ -611,17 +609,31 @@ def _valid_materialize_keys(self, features: List[str], allow_empty_key=False): return False return True - def materialize_features(self, settings: MaterializationSettings, execution_configurations: Union[SparkExecutionConfiguration ,Dict[str,str]] = {}, verbose: bool = False): + def materialize_features(self, settings: MaterializationSettings, execution_configurations: Union[SparkExecutionConfiguration ,Dict[str,str]] = {}, verbose: bool = False, allow_materialize_non_agg_feature: bool = False): """Materialize feature data Args: settings: Feature materialization settings execution_configurations: a dict that will be passed to spark job when the job starts up, i.e. the "spark configurations". Note that not all of the configuration will be honored since some of the configurations are managed by the Spark platform, such as Databricks or Azure Synapse. Refer to the [spark documentation](https://spark.apache.org/docs/latest/configuration.html) for a complete list of spark configurations. + allow_materialize_non_agg_feature: Materializing non-aggregated features (the features without WindowAggTransformation) doesn't output meaningful results so it's by default set to False, but if you really want to materialize non-aggregated features, set this to True. """ feature_list = settings.feature_names if len(feature_list) > 0 and not self._valid_materialize_keys(feature_list): raise RuntimeError(f"Invalid materialization features: {feature_list}, since they have different keys. Currently Feathr only supports materializing features of the same keys.") + if not allow_materialize_non_agg_feature: + # Check if there are non-aggregation features in the list + for fn in feature_list: + # Check over anchor features + for anchor in self.anchor_list: + for feature in anchor.features: + if feature.name == fn and not isinstance(feature.transform, WindowAggTransformation): + raise RuntimeError(f"Feature {fn} is not an aggregation feature. Currently Feathr only supports materializing aggregation features. If you want to materialize {fn}, please set allow_materialize_non_agg_feature to True.") + # Check over derived features + for feature in self.derived_feature_list: + if feature.name == fn and not isinstance(feature.transform, WindowAggTransformation): + raise RuntimeError(f"Feature {fn} is not an aggregation feature. Currently Feathr only supports materializing aggregation features. If you want to materialize {fn}, please set allow_materialize_non_agg_feature to True.") + # Collect secrets from sinks secrets = [] for sink in settings.sinks: @@ -631,7 +643,7 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf # produce materialization config for end in settings.get_backfill_cutoff_time(): settings.backfill_time.end = end - config = _to_materialization_config(settings) + config = _to_materialization_config(settings) config_file_name = "feature_gen_conf/auto_gen_config_{}.conf".format(end.timestamp()) config_file_path = os.path.join(self.local_workspace_dir, config_file_name) write_to_file(content=config, full_file_name=config_file_path) @@ -855,7 +867,7 @@ def get_features_from_registry(self, project_name: str) -> Dict[str, FeatureBase feature_dict[feature.name] = feature for feature in registry_derived_feature_list: feature_dict[feature.name] = feature - return feature_dict + return feature_dict def _reshape_config_str(self, config_str:str): if self.spark_runtime == 'local': diff --git a/feathr_project/feathr/constants.py b/feathr_project/feathr/constants.py index 6686f14ac..b2222e2b6 100644 --- a/feathr_project/feathr/constants.py +++ b/feathr_project/feathr/constants.py @@ -28,7 +28,11 @@ TYPEDEF_ARRAY_DERIVED_FEATURE=f"array" TYPEDEF_ARRAY_ANCHOR_FEATURE=f"array" -FEATHR_MAVEN_ARTIFACT="com.linkedin.feathr:feathr_2.12:0.8.0" +# Decouple Feathr MAVEN Version from Feathr Python SDK Version +import os +from feathr.version import __version__ +FEATHR_MAVEN_VERSION = os.environ.get("FEATHR_MAVEN_VERSION", __version__) +FEATHR_MAVEN_ARTIFACT=f"com.linkedin.feathr:feathr_2.12:{FEATHR_MAVEN_VERSION}" JOIN_CLASS_NAME="com.linkedin.feathr.offline.job.FeatureJoinJob" GEN_CLASS_NAME="com.linkedin.feathr.offline.job.FeatureGenJob" \ No newline at end of file diff --git a/feathr_project/feathr/definition/_materialization_utils.py b/feathr_project/feathr/definition/_materialization_utils.py index ef066b104..b49f7dced 100644 --- a/feathr_project/feathr/definition/_materialization_utils.py +++ b/feathr_project/feathr/definition/_materialization_utils.py @@ -10,6 +10,9 @@ def _to_materialization_config(settings: MaterializationSettings): endTime: "{{ settings.backfill_time.end.strftime('%Y-%m-%d %H:%M:%S') }}" endTimeFormat: "yyyy-MM-dd HH:mm:ss" resolution: DAILY + {% if settings.has_hdfs_sink == True %} + enableIncremental = true + {% endif %} output:[ {% for sink in settings.sinks %} {{sink.to_feature_config()}} diff --git a/feathr_project/feathr/definition/feature_derivations.py b/feathr_project/feathr/definition/feature_derivations.py index 84583654f..9205685ce 100644 --- a/feathr_project/feathr/definition/feature_derivations.py +++ b/feathr_project/feathr/definition/feature_derivations.py @@ -58,7 +58,7 @@ def to_feature_config(self) -> str: } {% endfor %} } - definition: {{derived_feature.transform.to_feature_config(False)}} + definition.sqlExpr: {{derived_feature.transform.to_feature_config(False)}} {{derived_feature.feature_type.to_feature_config()}} } """) diff --git a/feathr_project/feathr/definition/lookup_feature.py b/feathr_project/feathr/definition/lookup_feature.py index 647df37ce..2f1b80ccd 100644 --- a/feathr_project/feathr/definition/lookup_feature.py +++ b/feathr_project/feathr/definition/lookup_feature.py @@ -4,12 +4,13 @@ from jinja2 import Template from feathr.definition.dtype import FeatureType +from feathr.definition.feature_derivations import DerivedFeature from feathr.definition.feature import FeatureBase from feathr.definition.transformation import RowTransformation from feathr.definition.typed_key import DUMMY_KEY, TypedKey from feathr.definition.aggregation import Aggregation -class LookupFeature(FeatureBase): +class LookupFeature(DerivedFeature): """A lookup feature is a feature defined on top of two other features, i.e. using the feature value of the base feature as key, to lookup the feature value from the expansion feature. e.g. a lookup feature user_purchased_item_avg_price could be key-ed by user_id, and computed by: base feature is user_purchased_item_ids. For a given user_id, it returns the item ids purchased by the user. @@ -36,7 +37,8 @@ def __init__(self, key: Optional[Union[TypedKey, List[TypedKey]]] = [DUMMY_KEY], registry_tags: Optional[Dict[str, str]] = None, ): - super(LookupFeature, self).__init__(name, feature_type, key=key, registry_tags=registry_tags) + super(LookupFeature, self).__init__(name, feature_type, input_features=[base_feature, expansion_feature], + transform="", key=key, registry_tags=registry_tags) self.base_feature = base_feature self.expansion_feature = expansion_feature self.aggregation = aggregation diff --git a/feathr_project/feathr/definition/materialization_settings.py b/feathr_project/feathr/definition/materialization_settings.py index 8cdc2fc71..27b644139 100644 --- a/feathr_project/feathr/definition/materialization_settings.py +++ b/feathr_project/feathr/definition/materialization_settings.py @@ -32,7 +32,10 @@ def __init__(self, name: str, sinks: List[Sink], feature_names: List[str], backf now = datetime.now() self.backfill_time = backfill_time if backfill_time else BackfillTime(start=now, end=now, step=timedelta(days=1)) for sink in sinks: - if isinstance(sink, RedisSink): + if isinstance(sink, HdfsSink): + self.has_hdfs_sink = True + sink.aggregation_features = feature_names + elif isinstance(sink, RedisSink): sink.aggregation_features = feature_names self.sinks = sinks self.feature_names = feature_names diff --git a/feathr_project/feathr/definition/sink.py b/feathr_project/feathr/definition/sink.py index a23718a44..71c406561 100644 --- a/feathr_project/feathr/definition/sink.py +++ b/feathr_project/feathr/definition/sink.py @@ -103,25 +103,35 @@ def to_argument(self): class HdfsSink(Sink): """Offline Hadoop HDFS-compatible(HDFS, delta lake, Azure blog storage etc) sink that is used to store feature data. - The result is in AVRO format. + The result is in AVRO format. + + Incremental aggregation is enabled by default when using HdfsSink. Use incremental aggregation will significantly expedite the WindowAggTransformation feature calculation. + For example, aggregation sum of a feature F within a 180-day window at day T can be expressed as: F(T) = F(T - 1)+DirectAgg(T-1)-DirectAgg(T - 181). + Once a SNAPSHOT of the first day is generated, the calculation for the following days can leverage it. Attributes: output_path: output path + store_name: the folder name under the base "path". Used especially for the current dataset to support 'Incremental' aggregation. + """ - def __init__(self, output_path: str) -> None: + def __init__(self, output_path: str, store_name: Optional[str]="df0") -> None: self.output_path = output_path - + self.store_name = store_name # Sample generated HOCON config: # operational: { # name: testFeatureGen # endTime: 2019-05-01 # endTimeFormat: "yyyy-MM-dd" # resolution: DAILY + # enableIncremental = true # output:[ # { # name: HDFS + # outputFormat: RAW_DATA # params: { # path: "/user/featureGen/hdfsResult/" + # features: [mockdata_a_ct_gen, mockdata_a_sample_gen] + # storeName: "yyyy/MM/dd" # } # } # ] @@ -132,11 +142,15 @@ def to_feature_config(self) -> str: tm = Template(""" { name: HDFS + outputFormat: RAW_DATA params: { path: "{{sink.output_path}}" {% if sink.aggregation_features %} features: [{{','.join(sink.aggregation_features)}}] {% endif %} + {% if sink.store_name %} + storeName: "{{sink.store_name}}" + {% endif %} } } """) diff --git a/feathr_project/feathr/registry/_feature_registry_purview.py b/feathr_project/feathr/registry/_feature_registry_purview.py index 3210c8a87..7393acc90 100644 --- a/feathr_project/feathr/registry/_feature_registry_purview.py +++ b/feathr_project/feathr/registry/_feature_registry_purview.py @@ -3,6 +3,7 @@ import inspect import itertools import os +import re import sys import ast import types @@ -44,6 +45,25 @@ from feathr.constants import * +def _to_snake(d, level: int = 0): + """ + Convert `string`, `list[string]`, or all keys in a `dict` into snake case + The maximum length of input string or list is 100, or it will be truncated before being processed, for dict, the exception will be thrown if it has more than 100 keys. + the maximum nested level is 10, otherwise the exception will be thrown + """ + if level >= 10: + raise ValueError("Too many nested levels") + if isinstance(d, str): + d = d[:100] + return re.sub(r'(? 100: + raise ValueError("Dict has too many keys") + return {_to_snake(a, level + 1): _to_snake(b, level + 1) if isinstance(b, (dict, list)) else b for a, b in d.items()} + + class _PurviewRegistry(FeathrRegistry): """ Initializes the feature registry, doing the following: @@ -734,6 +754,32 @@ def upload_single_entity_to_purview(self,entity:Union[AtlasEntity,AtlasProcess]) The entity itself will also be modified, fill the GUID with real GUID in Purview. In order to avoid having concurrency issue, and provide clear guidance, this method only allows entity uploading once at a time. ''' + try: + """ + Try to find existing entity/process first, if found, return the existing entity's GUID + """ + id = self.get_entity_id(entity.qualifiedName) + response = self.purview_client.get_entity(id)['entities'][0] + j = entity.to_json() + if j["typeName"] == response["typeName"]: + if j["typeName"] == "Process": + if response["attributes"]["qualifiedName"] != j["attributes"]["qualifiedName"]: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + else: + if "type" in response['attributes'] and response["typeName"] in (TYPEDEF_ANCHOR_FEATURE, TYPEDEF_DERIVED_FEATURE): + conf = ConfigFactory.parse_string(response['attributes']['type']) + response['attributes']['type'] = dict(conf) + keys = set([_to_snake(key) for key in j["attributes"].keys()]) - set(["qualified_name"]) + keys.add("qualifiedName") + for k in keys: + if response["attributes"][k] != j["attributes"][k]: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + return response["guid"] + else: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + except AtlasException as e: + pass + try: entity.lastModifiedTS="0" result = self.purview_client.upload_entities([entity]) @@ -1382,11 +1428,11 @@ def _get_transformation_from_dict(self, input: Dict) -> FeatureType: if 'transformExpr' in input: # it's ExpressionTransformation return ExpressionTransformation(input['transformExpr']) - elif 'def_expr' in input: - agg_expr=input['def_expr'] if 'def_expr' in input else None - agg_func=input['agg_func']if 'agg_func' in input else None + elif 'def_expr' in input or 'defExpr' in input: + agg_expr=input['def_expr'] if 'def_expr' in input else (input['defExpr'] if 'defExpr' in input else None) + agg_func=input['agg_func']if 'agg_func' in input else (input['aggFunc'] if 'aggFunc' in input else None) window=input['window']if 'window' in input else None - group_by=input['group_by']if 'group_by' in input else None + group_by=input['group_by']if 'group_by' in input else (input['groupBy'] if 'groupBy' in input else None) filter=input['filter']if 'filter' in input else None limit=input['limit']if 'limit' in input else None return WindowAggTransformation(agg_expr, agg_func, window, group_by, filter, limit) diff --git a/feathr_project/feathr/spark_provider/_abc.py b/feathr_project/feathr/spark_provider/_abc.py index ff82e27ab..e82b42353 100644 --- a/feathr_project/feathr/spark_provider/_abc.py +++ b/feathr_project/feathr/spark_provider/_abc.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod +from typing import Dict, List, Optional, Tuple -from typing import Any, Dict, List, Optional, Tuple class SparkJobLauncher(ABC): diff --git a/feathr_project/feathr/spark_provider/_databricks_submission.py b/feathr_project/feathr/spark_provider/_databricks_submission.py index ac4d7f7fb..cfff0180e 100644 --- a/feathr_project/feathr/spark_provider/_databricks_submission.py +++ b/feathr_project/feathr/spark_provider/_databricks_submission.py @@ -69,8 +69,10 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): """ src_parse_result = urlparse(local_path_or_http_path) file_name = os.path.basename(local_path_or_http_path) - # returned paths for the uploaded file - returned_path = os.path.join(self.databricks_work_dir, file_name) + # returned paths for the uploaded file. Note that we cannot use os.path.join here, since in Windows system it will yield paths like this: + # dbfs:/feathrazure_cijob_snowflake_9_30_157692\auto_generated_derived_features.conf, where the path sep is mixed, and won't be able to be parsed by databricks. + # so we force the path to be Linux style here. + cloud_dest_path = self.databricks_work_dir + "/" + file_name if src_parse_result.scheme.startswith('http'): with urlopen(local_path_or_http_path) as f: # use REST API to avoid local temp file @@ -78,14 +80,14 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): files = {'file': data} # for DBFS APIs, see: https://docs.microsoft.com/en-us/azure/databricks/dev-tools/api/latest/dbfs r = requests.post(url=self.workspace_instance_url+'/api/2.0/dbfs/put', - headers=self.auth_headers, files=files, data={'overwrite': 'true', 'path': returned_path}) + headers=self.auth_headers, files=files, data={'overwrite': 'true', 'path': cloud_dest_path}) logger.info('{} is downloaded and then uploaded to location: {}', - local_path_or_http_path, returned_path) + local_path_or_http_path, cloud_dest_path) elif src_parse_result.scheme.startswith('dbfs'): # passed a cloud path logger.info( 'Skip uploading file {} as the file starts with dbfs:/', local_path_or_http_path) - returned_path = local_path_or_http_path + cloud_dest_path = local_path_or_http_path elif src_parse_result.scheme.startswith(('wasb','s3','gs')): # if the path starts with a location that's not a local path logger.error("File {} cannot be downloaded. Please upload the file to dbfs manually.", local_path_or_http_path) @@ -96,27 +98,29 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): logger.info("Uploading folder {}", local_path_or_http_path) dest_paths = [] for item in Path(local_path_or_http_path).glob('**/*.conf'): - returned_path = self.upload_local_file(item.resolve()) - dest_paths.extend([returned_path]) - returned_path = ','.join(dest_paths) + cloud_dest_path = self._upload_local_file_to_workspace(item.resolve()) + dest_paths.extend([cloud_dest_path]) + cloud_dest_path = ','.join(dest_paths) else: - returned_path = self.upload_local_file(local_path_or_http_path) - return returned_path + cloud_dest_path = self._upload_local_file_to_workspace(local_path_or_http_path) + return cloud_dest_path - def upload_local_file(self, local_path: str) -> str: + def _upload_local_file_to_workspace(self, local_path: str) -> str: """ Supports transferring file from a local path to cloud working storage. """ file_name = os.path.basename(local_path) - # returned paths for the uploaded file - returned_path = os.path.join(self.databricks_work_dir, file_name) + # returned paths for the uploaded file. Note that we cannot use os.path.join here, since in Windows system it will yield paths like this: + # dbfs:/feathrazure_cijob_snowflake_9_30_157692\auto_generated_derived_features.conf, where the path sep is mixed, and won't be able to be parsed by databricks. + # so we force the path to be Linux style here. + cloud_dest_path = self.databricks_work_dir + "/" + file_name # `local_path_or_http_path` will be either string or PathLib object, so normalize it to string local_path = str(local_path) try: - DbfsApi(self.api_client).cp(recursive=True, overwrite=True, src=local_path, dst=returned_path) + DbfsApi(self.api_client).cp(recursive=True, overwrite=True, src=local_path, dst=cloud_dest_path) except RuntimeError as e: - raise RuntimeError(f"The source path: {local_path}, or the destination path: {returned_path}, is/are not valid.") from e - return returned_path + raise RuntimeError(f"The source path: {local_path}, or the destination path: {cloud_dest_path}, is/are not valid.") from e + return cloud_dest_path def submit_feathr_job(self, job_name: str, main_jar_path: str, main_class_name: str, arguments: List[str], python_files: List[str], reference_files_path: List[str] = [], job_tags: Dict[str, str] = None, configuration: Dict[str, str] = {}, properties: Dict[str, str] = {}): """ diff --git a/feathr_project/feathr/spark_provider/_localspark_submission.py b/feathr_project/feathr/spark_provider/_localspark_submission.py index 3b24fd513..afed9683d 100644 --- a/feathr_project/feathr/spark_provider/_localspark_submission.py +++ b/feathr_project/feathr/spark_provider/_localspark_submission.py @@ -1,129 +1,125 @@ -import time from datetime import datetime import json import os from pathlib import Path -from typing import Dict, List, Optional +from shlex import split +from subprocess import STDOUT, Popen +import time +from typing import Any, Dict, List, Optional -from feathr.spark_provider._abc import SparkJobLauncher from loguru import logger - from pyspark import * -from subprocess import TimeoutExpired, STDOUT, Popen -from shlex import split from feathr.constants import FEATHR_MAVEN_ARTIFACT +from feathr.spark_provider._abc import SparkJobLauncher +class _FeathrLocalSparkJobLauncher(SparkJobLauncher): + """Class to interact with local Spark. This class is not intended to be used in Production environments. + It is intended to be used for testing and development purposes. No authentication is required to use this class. -class _FeathrDLocalSparkJobLauncher(SparkJobLauncher): - """Class to interact with local Spark - This class is not intended to be used in Production environments. - It is intended to be used for testing and development purposes. - No authentication is required to use this class. - Args: - workspace_path (str): Path to the workspace + Args: + workspace_path (str): Path to the workspace """ + def __init__( self, workspace_path: str, master: str = None, - debug_folder:str = "debug", - clean_up:bool = True, - retry:int = 3, - retry_sec:int = 5, + debug_folder: str = "debug", + clean_up: bool = True, + retry: int = 3, + retry_sec: int = 5, ): - """Initialize the Local Spark job launcher - """ - self.workspace_path = workspace_path, + """Initialize the Local Spark job launcher""" + self.workspace_path = (workspace_path,) self.debug_folder = debug_folder self.spark_job_num = 0 self.clean_up = clean_up self.retry = retry self.retry_sec = retry_sec self.packages = self._get_default_package() - self.master = master + self.master = master or "local[*]" def upload_or_get_cloud_path(self, local_path_or_http_path: str): """For Local Spark Case, no need to upload to cloud workspace.""" return local_path_or_http_path - def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_class_name: str = None, arguments: List[str] = None, - python_files: List[str]= None, configuration: Dict[str, str] = {}, properties: Dict[str, str] = {}, reference_files_path: List[str] = None, job_tags: Dict[str, str] = None): - """ - Submits the Feathr job to local spark, using subprocess args. - - reference files: put everything there and the function will automatically categorize them based on the - extension name to either the "files" argument in the Livy API, or the "jars" argument in the Livy API. The - path can be local path and this function will automatically upload the function to the corresponding azure - storage - - Also, note that the Spark application will automatically run on YARN cluster mode. You cannot change it if + def submit_feathr_job( + self, + job_name: str, + main_jar_path: str, + main_class_name: str, + arguments: List[str] = None, + python_files: List[str] = None, + configuration: Dict[str, str] = {}, + properties: Dict[str, str] = {}, + **_, + ) -> Any: + """Submits the Feathr job to local spark, using subprocess args. + Note that the Spark application will automatically run on YARN cluster mode. You cannot change it if you are running with Azure Synapse. Args: - job_name (str): name of the job - main_jar_path (str): main file paths, usually your main jar file - main_class_name (str): name of your main class - arguments (str): all the arguments you want to pass into the spark job - configuration (Dict[str, str]): Additional configs for the spark job - python_files (List[str]): required .zip, .egg, or .py files of spark job - properties (Dict[str, str]): Additional System Properties for the spark job - job_tags (str): not used in local spark mode - reference_files_path (str): not used in local spark mode + job_name: name of the job + main_jar_path: main file paths, usually your main jar file + main_class_name: name of your main class + arguments: all the arguments you want to pass into the spark job + python_files: required .zip, .egg, or .py files of spark job + configuration: Additional configs for the spark job + properties: System properties configuration + **_: Not used arguments in local spark mode, such as reference_files_path and job_tags """ - logger.warning(f"Local Spark Mode only support basic params right now and should be used only for testing purpose.") - self.cmd_file, self.log_path = self._get_debug_file_name(self.debug_folder, prefix = job_name) - args = self._init_args(master = self.master, job_name=job_name) + logger.warning( + f"Local Spark Mode only support basic params right now and should be used only for testing purpose." + ) + self.cmd_file, self.log_path = self._get_debug_file_name(self.debug_folder, prefix=job_name) - if properties: - arguments.extend(["--system-properties", json.dumps(properties)]) + # Get conf and package arguments + cfg = configuration.copy() if configuration else {} + maven_dependency = f"{cfg.pop('spark.jars.packages', self.packages)},{FEATHR_MAVEN_ARTIFACT}" + spark_args = self._init_args(job_name=job_name, confs=cfg) - if configuration: - cfg = configuration.copy() # We don't want to mess up input parameters - else: - cfg = {} - if not main_jar_path: # We don't have the main jar, use Maven - # Add Maven dependency to the job configuration - if "spark.jars.packages" in cfg: - cfg["spark.jars.packages"] = ",".join( - [cfg["spark.jars.packages"], FEATHR_MAVEN_ARTIFACT]) - else: - cfg["spark.jars.packages"] = ",".join([self.packages, FEATHR_MAVEN_ARTIFACT]) - if not python_files: # This is a JAR job # Azure Synapse/Livy doesn't allow JAR job starts from Maven directly, we must have a jar file uploaded. # so we have to use a dummy jar as the main file. logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") # Use the no-op jar as the main file - # This is a dummy jar which contains only one `org.example.Noop` class with one empty `main` function which does nothing + # This is a dummy jar which contains only one `org.example.Noop` class with one empty `main` function + # which does nothing current_dir = Path(__file__).parent.resolve() main_jar_path = os.path.join(current_dir, "noop-1.0.jar") - args.extend(["--packages", cfg["spark.jars.packages"],"--class", main_class_name, main_jar_path]) + spark_args.extend(["--packages", maven_dependency, "--class", main_class_name, main_jar_path]) else: - args.extend(["--packages", cfg["spark.jars.packages"]]) - # This is a PySpark job, no more things to + spark_args.extend(["--packages", maven_dependency]) + # This is a PySpark job, no more things to if python_files.__len__() > 1: - args.extend(["--py-files", ",".join(python_files[1:])]) + spark_args.extend(["--py-files", ",".join(python_files[1:])]) print(python_files) - args.append(python_files[0]) + spark_args.append(python_files[0]) else: - args.extend(["--class", main_class_name, main_jar_path]) + spark_args.extend(["--class", main_class_name, main_jar_path]) + + if arguments: + spark_args.extend(arguments) - cmd = " ".join(args) + " " + " ".join(arguments) + if properties: + spark_args.extend(["--system-properties", json.dumps(properties)]) + + cmd = " ".join(spark_args) - log_append = open(f"{self.log_path}_{self.spark_job_num}.txt" , "a") + log_append = open(f"{self.log_path}_{self.spark_job_num}.txt", "a") proc = Popen(split(cmd), shell=False, stdout=log_append, stderr=STDOUT) logger.info(f"Detail job stdout and stderr are in {self.log_path}.") self.spark_job_num += 1 with open(self.cmd_file, "a") as c: - c.write(" ".join(proc.args)) - c.write("\n") + c.write(" ".join(proc.args)) + c.write("\n") self.latest_spark_proc = proc @@ -132,9 +128,8 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_clas return proc def wait_for_completion(self, timeout_seconds: Optional[float] = 500) -> bool: - """ - this function track local spark job commands and process status. - files will be write into `debug` folder under your workspace. + """This function track local spark job commands and process status. + Files will be write into `debug` folder under your workspace. """ logger.info(f"{self.spark_job_num} local spark job(s) in this Launcher, only the latest will be monitored.") logger.info(f"Please check auto generated spark command in {self.cmd_file} and detail logs in {self.log_path}.") @@ -143,12 +138,15 @@ def wait_for_completion(self, timeout_seconds: Optional[float] = 500) -> bool: start_time = time.time() retry = self.retry - log_read = open(f"{self.log_path}_{self.spark_job_num-1}.txt" , "r") + log_read = open(f"{self.log_path}_{self.spark_job_num-1}.txt", "r") while proc.poll() is None and (((timeout_seconds is None) or (time.time() - start_time < timeout_seconds))): time.sleep(1) try: if retry < 1: - logger.warning(f"Spark job has hang for {self.retry * self.retry_sec} seconds. latest msg is {last_line}. please check {log_read.name}") + logger.warning( + f"Spark job has hang for {self.retry * self.retry_sec} seconds. latest msg is {last_line}. \ + Please check {log_read.name}" + ) if self.clean_up: self._clean_up() proc.wait() @@ -168,22 +166,28 @@ def wait_for_completion(self, timeout_seconds: Optional[float] = 500) -> bool: retry -= 1 job_duration = time.time() - start_time - log_read.close() + log_read.close() if proc.returncode == None: - logger.warning(f"Spark job with pid {self.latest_spark_proc.pid} not completed after {timeout_seconds} sec time out setting, please check.") + logger.warning( + f"Spark job with pid {self.latest_spark_proc.pid} not completed after {timeout_seconds} sec \ + time out setting. Please check." + ) if self.clean_up: self._clean_up() proc.wait() return True elif proc.returncode == 1: - logger.warning(f"Spark job with pid {self.latest_spark_proc.pid} is not successful, please check.") + logger.warning(f"Spark job with pid {self.latest_spark_proc.pid} is not successful. Please check.") return False else: - logger.info(f"Spark job with pid {self.latest_spark_proc.pid} finished in: {int(job_duration)} seconds with returncode {proc.returncode}") + logger.info( + f"Spark job with pid {self.latest_spark_proc.pid} finished in: {int(job_duration)} seconds \ + with returncode {proc.returncode}" + ) return True - def _clean_up(self, proc:Popen = None): + def _clean_up(self, proc: Popen = None): logger.warning(f"Terminate the spark job due to as clean_up is set to True.") if not proc: self.latest_spark_proc.terminate() @@ -194,30 +198,35 @@ def get_status(self) -> str: """Get the status of the job, only a placeholder for local spark""" return self.latest_spark_proc.returncode - def _init_args(self, master:str, job_name:str): - if master is None: - master = "local[*]" - logger.info(f"Spark job: {job_name} is running on local spark with master: {master}.") + def _init_args(self, job_name: str, confs: Dict[str, str]) -> List[str]: + logger.info(f"Spark job: {job_name} is running on local spark with master: {self.master}.") args = [ "spark-submit", - "--master",master, - "--name",job_name, - "--conf", "spark.hadoop.fs.wasbs.impl=org.apache.hadoop.fs.azure.NativeAzureFileSystem", - "--conf", "spark.hadoop.fs.wasbs=org.apache.hadoop.fs.azure.NativeAzureFileSystem", + "--master", + self.master, + "--name", + job_name, + "--conf", + "spark.hadoop.fs.wasbs.impl=org.apache.hadoop.fs.azure.NativeAzureFileSystem", + "--conf", + "spark.hadoop.fs.wasbs=org.apache.hadoop.fs.azure.NativeAzureFileSystem", ] + + for k, v in confs.items(): + args.extend(["--conf", f"{k}={v}"]) + return args - def _get_debug_file_name(self, debug_folder: str = "debug", prefix:str = None): - """ - auto generated command will be write into cmd file - spark job output will be write into log path with job number as suffix + def _get_debug_file_name(self, debug_folder: str = "debug", prefix: str = None): + """Auto generated command will be write into cmd file. + Spark job output will be write into log path with job number as suffix. """ prefix += datetime.now().strftime("%Y%m%d%H%M%S") debug_path = os.path.join(debug_folder, prefix) print(debug_path) if not os.path.exists(debug_path): - os.makedirs(debug_path) + os.makedirs(debug_path) cmd_file = os.path.join(debug_path, f"command.sh") log_path = os.path.join(debug_path, f"log") @@ -227,7 +236,7 @@ def _get_debug_file_name(self, debug_folder: str = "debug", prefix:str = None): def _get_default_package(self): # default packages of Feathr Core, requires manual update when new dependency introduced or package updated. # TODO: automate this process, e.g. read from pom.xml - # TODO: dynamical modularization: add package only when it's used in the job, e.g. data source dependencies. + # TODO: dynamical modularization: add package only when it's used in the job, e.g. data source dependencies. packages = [] packages.append("org.apache.spark:spark-avro_2.12:3.3.0") packages.append("com.microsoft.sqlserver:mssql-jdbc:10.2.0.jre8") @@ -236,7 +245,7 @@ def _get_default_package(self): packages.append("com.fasterxml.jackson.core:jackson-databind:2.12.6.1") packages.append("org.apache.hadoop:hadoop-mapreduce-client-core:2.7.7") packages.append("org.apache.hadoop:hadoop-common:2.7.7") - packages.append("org.apache.hadoop:hadoop-azure:3.2.0") + packages.append("org.apache.hadoop:hadoop-azure:3.2.0") packages.append("org.apache.avro:avro:1.8.2,org.apache.xbean:xbean-asm6-shaded:4.10") packages.append("org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.3") packages.append("com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.21") diff --git a/feathr_project/feathr/spark_provider/_synapse_submission.py b/feathr_project/feathr/spark_provider/_synapse_submission.py index b72acdc42..53df12c13 100644 --- a/feathr_project/feathr/spark_provider/_synapse_submission.py +++ b/feathr_project/feathr/spark_provider/_synapse_submission.py @@ -114,6 +114,7 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_clas if not main_jar_path: # We don't have the main jar, use Maven # Add Maven dependency to the job configuration + logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") if "spark.jars.packages" in cfg: cfg["spark.jars.packages"] = ",".join( [cfg["spark.jars.packages"], FEATHR_MAVEN_ARTIFACT]) @@ -124,7 +125,6 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str = None, main_clas # This is a JAR job # Azure Synapse/Livy doesn't allow JAR job starts from Maven directly, we must have a jar file uploaded. # so we have to use a dummy jar as the main file. - logger.info(f"Main JAR file is not set, using default package '{FEATHR_MAVEN_ARTIFACT}' from Maven") # Use the no-op jar as the main file # This is a dummy jar which contains only one `org.example.Noop` class with one empty `main` function which does nothing current_dir = pathlib.Path(__file__).parent.resolve() @@ -325,7 +325,7 @@ def create_spark_batch_job(self, job_name, main_file, class_name=None, def get_driver_log(self, job_id) -> str: # @see: https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/connect-monitor-azure-synapse-spark-application-level-metrics app_id = self.get_spark_batch_job(job_id).app_id - url = "%s/sparkhistory/api/v1/sparkpools/%s/livyid/%s/applications/%s/driverlog/stdout/?isDownload=true" % (self._synapse_dev_url, self._spark_pool_name, job_id, app_id) + url = "%s/sparkhistory/api/v1/sparkpools/%s/livyid/%s/applications/%s/driverlog/stderr/?isDownload=true" % (self._synapse_dev_url, self._spark_pool_name, job_id, app_id) token = self._credential.get_token("https://dev.azuresynapse.net/.default").token req = urllib.request.Request(url=url, headers={"authorization": "Bearer %s" % token}) resp = urllib.request.urlopen(req) diff --git a/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py b/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py index ca7114343..55756ba3d 100644 --- a/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py +++ b/feathr_project/feathr/udf/_preprocessing_pyudf_manager.py @@ -1,12 +1,15 @@ +import ast import inspect import os +import pickle from pathlib import Path from typing import List, Optional, Union -import pickle -from feathr.definition.anchor import FeatureAnchor + from jinja2 import Template + +from feathr.definition.anchor import FeatureAnchor from feathr.definition.source import HdfsSource -import ast + # Some metadata that are only needed by Feathr FEATHR_PYSPARK_METADATA = 'generated_feathr_pyspark_metadata' @@ -42,7 +45,7 @@ def build_anchor_preprocessing_metadata(anchor_list: List[FeatureAnchor], local_ # delete the file if it already exists to avoid caching previous results for f in [client_udf_repo_path, metadata_path, pyspark_driver_path]: if os.path.exists(f): - os.remove(f) + os.remove(f) for anchor in anchor_list: # only support batch source preprocessing for now. @@ -73,23 +76,29 @@ def build_anchor_preprocessing_metadata(anchor_list: List[FeatureAnchor], local_ with open(feathr_pyspark_metadata_abs_path, 'wb') as file: pickle.dump(features_with_preprocessing, file) + @staticmethod - def _parse_function_str_for_name(source: str) -> str: - """ - Use AST to parse the functions and get the name out. + def _parse_function_str_for_name(fn_str: str) -> str: + """Use AST to parse the function string and get the name out. + + Args: + fn_str: Function code in string. + + Returns: + Name of the function. """ - if source is None: + if not fn_str: return None - tree = ast.parse(source) + + tree = ast.parse(fn_str) + + # tree.body contains a list of function definition objects parsed from the input string. + # Currently, we only accept a single function. if len(tree.body) != 1 or not isinstance(tree.body[0], ast.FunctionDef): - raise ValueError('provided code fragment is not a single function') - code = compile(source=tree, filename='custom.py',mode= 'exec') - # https://docs.python.org/3/library/inspect.html see the inspect module for more details - # tuple of names other than arguments and function locals. Assume there will be only one function, so will return the first as the name - for ele in code.co_consts: - # find the first object, that is the str, this will be the name of the function - if isinstance(ele, str): - return ele + raise ValueError("provided code fragment is not a single function") + + # Get the function name from the function definition. + return tree.body[0].name @staticmethod @@ -174,7 +183,7 @@ def prepare_pyspark_udf_files(feature_names: List[str], local_workspace_dir): client_udf_repo_path = os.path.join(local_workspace_dir, FEATHR_CLIENT_UDF_FILE_NAME) # write pyspark_driver_template_abs_path and then client_udf_repo_path filenames = [pyspark_driver_template_abs_path, client_udf_repo_path] - + with open(pyspark_driver_path, 'w') as outfile: for fname in filenames: with open(fname) as infile: diff --git a/feathr_project/feathr/version.py b/feathr_project/feathr/version.py new file mode 100644 index 000000000..145f98940 --- /dev/null +++ b/feathr_project/feathr/version.py @@ -0,0 +1 @@ +__version__ = "0.9.0-rc2" \ No newline at end of file diff --git a/feathr_project/pyproject.toml b/feathr_project/pyproject.toml index f8d897579..693233dc2 100644 --- a/feathr_project/pyproject.toml +++ b/feathr_project/pyproject.toml @@ -1,6 +1,17 @@ +[tool.black] +line-length = 120 +target_version = ['py38'] + +[tool.isort] +profile = "black" +line_length = 120 +known_first_party = ['feathr'] +force_sort_within_sections = true +multi_line_output = 3 + [build-system] requires = [ "setuptools", "wheel" ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" diff --git a/feathr_project/setup.py b/feathr_project/setup.py index 373709395..157ca8068 100644 --- a/feathr_project/setup.py +++ b/feathr_project/setup.py @@ -1,3 +1,5 @@ +import sys +import os from setuptools import setup, find_packages from pathlib import Path @@ -5,9 +7,36 @@ root_path = Path(__file__).resolve().parent.parent long_description = (root_path / "docs/README.md").read_text(encoding="utf8") +try: + exec(open("feathr/version.py").read()) +except IOError: + print("Failed to load Feathr version file for packaging.", + file=sys.stderr) + sys.exit(-1) + +VERSION = __version__ # noqa +os.environ["FEATHR_VERSION]"] = VERSION + +extras_require=dict( + dev=[ + "black>=22.1.0", # formatter + "isort", # sort import statements + "pytest>=7", + "pytest-xdist", + "pytest-mock>=3.8.1", + ], + notebook=[ + "jupyter==1.0.0", + "matplotlib==3.6.1", + "papermill>=2.1.2,<3", # to test run notebooks + "scrapbook>=0.5.0,<1.0.0", # to scrap notebook outputs + ], +) +extras_require["all"] = list(set(sum([*extras_require.values()], []))) + setup( name='feathr', - version='0.8.0', + version=VERSION, long_description=long_description, long_description_content_type="text/markdown", author_email="feathr-technical-discuss@lists.lfaidata.foundation", @@ -20,7 +49,7 @@ include_package_data=True, # consider install_requires=[ - 'click<=8.1.3', + "click<=8.1.3", "py4j<=0.10.9.7", "loguru<=0.6.0", "pandas<=1.5.0", @@ -35,10 +64,9 @@ "pyarrow<=9.0.0", "pyspark>=3.1.2", "python-snappy<=0.6.1", - # fixing https://github.com/feathr-ai/feathr/issues/687 - "deltalake<=0.5.8", + "deltalake>=0.6.2", "graphlib_backport<=1.0.3", - "protobuf==3.*", + "protobuf<=3.19.4,>=3.0.0", "confluent-kafka<=1.9.2", "databricks-cli<=0.17.3", "avro<=1.11.1", @@ -52,12 +80,18 @@ # https://github.com/Azure/azure-sdk-for-python/pull/22891 # using a version lower than that to workaround this issue. "azure-core<=1.22.1", + # azure-core 1.22.1 is dependent on msrest==0.6.21, if an environment(AML) has a different version of azure-core (say 1.24.0), + # it brings a different version of msrest(0.7.0) which is incompatible with azure-core==1.22.1. Hence we need to pin it. + # See this for more details: https://github.com/Azure/azure-sdk-for-python/issues/24765 + "msrest<=0.6.21", "typing_extensions>=4.2.0", "aws-secretsmanager-caching>=1.1.1.5", + "aws-requests-auth>=0.4.3" ], - tests_require=[ - 'pytest', + tests_require=[ # TODO: This has been depricated + "pytest", ], + extras_require=extras_require, entry_points={ 'console_scripts': ['feathr=feathrcli.cli:cli'] }, diff --git a/feathr_project/test/test_azure_kafka_e2e.py b/feathr_project/test/test_azure_kafka_e2e.py index 6c1a9b7d9..f680f695a 100644 --- a/feathr_project/test/test_azure_kafka_e2e.py +++ b/feathr_project/test/test_azure_kafka_e2e.py @@ -19,5 +19,5 @@ def test_feathr_kafa_streaming_features(): sinks=[redisSink], feature_names=['f_modified_streaming_count'] ) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) diff --git a/feathr_project/test/test_azure_snowflake_e2e.py b/feathr_project/test/test_azure_snowflake_e2e.py index c84aa9153..17474ab1b 100644 --- a/feathr_project/test/test_azure_snowflake_e2e.py +++ b/feathr_project/test/test_azure_snowflake_e2e.py @@ -30,7 +30,7 @@ def test_feathr_online_store_agg_features(): feature_names=['f_snowflake_call_center_division_name', 'f_snowflake_call_center_zipcode'], backfill_time=backfill_time) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) diff --git a/feathr_project/test/test_azure_spark_e2e.py b/feathr_project/test/test_azure_spark_e2e.py index d2aa0b032..ae7c1cab2 100644 --- a/feathr_project/test/test_azure_spark_e2e.py +++ b/feathr_project/test/test_azure_spark_e2e.py @@ -119,7 +119,7 @@ def test_feathr_online_store_non_agg_features(): feature_names=["f_gen_trip_distance", "f_gen_is_long_trip_distance", "f1", "f2", "f3", "f4", "f5", "f6"], backfill_time=backfill_time) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) @@ -183,7 +183,7 @@ def test_feathr_get_offline_features(): full_name="nyc_taxi.location_id") feature_query = FeatureQuery( - feature_list=["f_location_avg_fare"], key=location_id) + feature_list=["f_location_avg_fare", "f_trip_time_rounded"], key=location_id) settings = ObservationSettings( observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", event_timestamp_column="lpep_dropoff_datetime", @@ -309,9 +309,9 @@ def test_feathr_materialize_to_aerospike(): # os.chdir(test_workspace_dir) now = datetime.now() # set workspace folder by time; make sure we don't have write conflict if there are many CI tests running - os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) - os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) - + os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) + os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) + client = FeathrClient(config_path="feathr_config.yaml") batch_source = HdfsSource(name="nycTaxiBatchSource", path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", @@ -396,4 +396,4 @@ def test_feathr_materialize_to_aerospike(): if __name__ == "__main__": test_feathr_materialize_to_aerospike() test_feathr_get_offline_features_to_sql() - test_feathr_materialize_to_cosmosdb() \ No newline at end of file + test_feathr_materialize_to_cosmosdb() diff --git a/feathr_project/test/test_azure_spark_maven_e2e.py b/feathr_project/test/test_azure_spark_maven_e2e.py index b8e7cefb0..a2f214020 100644 --- a/feathr_project/test/test_azure_spark_maven_e2e.py +++ b/feathr_project/test/test_azure_spark_maven_e2e.py @@ -3,8 +3,12 @@ from pathlib import Path from feathr import (BackfillTime, MaterializationSettings) -from feathr import RedisSink +# from feathr import * from feathr.client import FeathrClient +from feathr.definition.dtype import ValueType +from feathr.definition.query_feature_list import FeatureQuery +from feathr.definition.settings import ObservationSettings +from feathr.definition.typed_key import TypedKey from test_fixture import (basic_test_setup, get_online_test_table_name) from test_utils.constants import Constants @@ -22,6 +26,35 @@ def test_feathr_online_store_agg_features(): # Maven package as the dependency and `noop.jar` as the main file client: FeathrClient = basic_test_setup(os.path.join(test_workspace_dir, "feathr_config_maven.yaml")) + + + location_id = TypedKey(key_column="DOLocationID", + key_column_type=ValueType.INT32, + description="location id in NYC", + full_name="nyc_taxi.location_id") + + feature_query = FeatureQuery( + feature_list=["f_location_avg_fare"], key=location_id) + settings = ObservationSettings( + observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", + event_timestamp_column="lpep_dropoff_datetime", + timestamp_format="yyyy-MM-dd HH:mm:ss") + + now = datetime.now() + # set output folder based on different runtime + if client.spark_runtime == 'databricks': + output_path = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), ".avro"]) + else: + output_path = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output','_', str(now.minute), '_', str(now.second), ".avro"]) + + + client.get_offline_features(observation_settings=settings, + feature_query=feature_query, + output_path=output_path) + + # assuming the job can successfully run; otherwise it will throw exception + client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) + return backfill_time = BackfillTime(start=datetime( 2020, 5, 20), end=datetime(2020, 5, 20), step=timedelta(days=1)) redisSink = RedisSink(table_name=online_test_table) @@ -51,4 +84,4 @@ def test_feathr_online_store_agg_features(): assert res['239'][0] != None assert res['239'][1] != None assert res['265'][0] != None - assert res['265'][1] != None \ No newline at end of file + assert res['265'][1] != None diff --git a/feathr_project/test/test_derived_features.py b/feathr_project/test/test_derived_features.py index f879553d7..ee10cd285 100644 --- a/feathr_project/test/test_derived_features.py +++ b/feathr_project/test/test_derived_features.py @@ -26,7 +26,7 @@ def test_single_key_derived_feature_to_config(): inputs: { user_embedding: {key: [user_id], feature: user_embedding} } - definition: "if_else(user_embedding, user_embedding, [])" + definition.sqlExpr: "if_else(user_embedding, user_embedding, [])" type: { type: TENSOR tensorCategory: DENSE @@ -58,7 +58,7 @@ def test_multikey_derived_feature_to_config(): user_embedding: {key: [user_id], feature: user_embedding} item_embedding: {key: [item_id], feature: item_embedding} } - definition: "similarity(user_embedding,item_embedding)" + definition.sqlExpr: "similarity(user_embedding,item_embedding)" type: { type: TENSOR tensorCategory: DENSE @@ -88,7 +88,7 @@ def test_derived_feature_to_config_with_alias(): viewer_embedding: {key: [viewer], feature: user_embedding} viewee_embedding: {key: [viewee], feature: user_embedding} } - definition: "distance(viewer_embedding, viewee_embedding)" + definition.sqlExpr: "distance(viewer_embedding, viewee_embedding)" type: { type: TENSOR tensorCategory: DENSE @@ -129,7 +129,7 @@ def test_multi_key_derived_feature_to_config_with_alias(): viewer_viewee_distance: {key: [viewer, viewee], feature: viewer_viewee_distance} viewee_viewer_distance: {key: [viewee, viewer], feature: viewer_viewee_distance} } - definition: "viewer_viewee_distance + viewee_viewer_distance" + definition.sqlExpr: "viewer_viewee_distance + viewee_viewer_distance" type: { type: TENSOR tensorCategory: DENSE @@ -159,7 +159,7 @@ def test_derived_feature_on_multikey_anchored_feature_to_config(): inputs: { user_embedding: {key: [viewer, viewee], feature: user_embedding} } - definition: "if_else(user_embedding, user_embedding, [])" + definition.sqlExpr: "if_else(user_embedding, user_embedding, [])" type: { type: TENSOR tensorCategory: DENSE diff --git a/feathr_project/test/test_feature_materialization.py b/feathr_project/test/test_feature_materialization.py index 62b84d367..e8100578c 100644 --- a/feathr_project/test/test_feature_materialization.py +++ b/feathr_project/test/test_feature_materialization.py @@ -61,12 +61,17 @@ def test_feature_materialization_offline_config(): endTime: "2020-05-20 00:00:00" endTimeFormat: "yyyy-MM-dd HH:mm:ss" resolution: DAILY + enableIncremental = true output:[ { name: HDFS + outputFormat: RAW_DATA params: { path: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/demo_data/output/hdfs_test.avro" + features: [f_location_avg_fare,f_location_max_fare] + storeName: "df0" } + } ] } @@ -231,7 +236,7 @@ def test_delete_feature_from_redis(): "f_day_of_week" ], backfill_time=backfill_time) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) diff --git a/feathr_project/test/test_feature_registry.py b/feathr_project/test/test_feature_registry.py index 5f2fea7d4..86db93440 100644 --- a/feathr_project/test/test_feature_registry.py +++ b/feathr_project/test/test_feature_registry.py @@ -59,18 +59,6 @@ def test_feathr_register_features_e2e(self): # Sync workspace from registry, will get all conf files back client.get_features_from_registry(client.project_name) - feature_query = FeatureQuery( - feature_list=["f_location_avg_fare", "f_trip_time_rounded", "f_is_long_trip_distance"], - key=TypedKey(key_column="DOLocationID",key_column_type=ValueType.INT32)) - settings = ObservationSettings( - observation_path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04_with_index.csv", - event_timestamp_column="lpep_dropoff_datetime", - timestamp_format="yyyy-MM-dd HH:mm:ss") - client.get_offline_features(observation_settings=settings, - feature_query=feature_query, - output_path=output_path) - client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) - def test_feathr_register_features_partially(self): """ This test will register full set of features into one project, then register another project in two partial registrations. diff --git a/feathr_project/test/test_lookup_feature.py b/feathr_project/test/test_lookup_feature.py index ffdb9a686..82fe385a7 100644 --- a/feathr_project/test/test_lookup_feature.py +++ b/feathr_project/test/test_lookup_feature.py @@ -1,6 +1,7 @@ from feathr import Aggregation from feathr import Feature from feathr import LookupFeature +from feathr import DerivedFeature from feathr import FLOAT, FLOAT_VECTOR, ValueType, INT32_VECTOR from feathr import TypedKey @@ -39,4 +40,5 @@ def test_single_key_lookup_feature_to_config(): } }""" assert_config_equals(lookup_feature.to_feature_config(), lookup_feature_config) + assert(isinstance(lookup_feature, DerivedFeature)) \ No newline at end of file diff --git a/feathr_project/test/test_pyduf_preprocessing_e2e.py b/feathr_project/test/test_pyduf_preprocessing_e2e.py index 9ac9c1917..83ace12ea 100644 --- a/feathr_project/test/test_pyduf_preprocessing_e2e.py +++ b/feathr_project/test/test_pyduf_preprocessing_e2e.py @@ -103,7 +103,7 @@ def test_non_swa_feature_gen_with_offline_preprocessing(): "f_day_of_week" ], backfill_time=backfill_time) - client.materialize_features(settings) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) # just assume the job is successful without validating the actual result in Redis. Might need to consolidate # this part with the test_feathr_online_store test case client.wait_job_to_finish(timeout_sec=Constants.SPARK_JOB_TIMEOUT_SECONDS) diff --git a/feathr_project/test/test_user_workspace/feathr_config.yaml b/feathr_project/test/test_user_workspace/feathr_config.yaml index e67c803ef..94fac6a23 100644 --- a/feathr_project/test/test_user_workspace/feathr_config.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config.yaml @@ -82,7 +82,7 @@ spark_config: # Feathr Job configuration. Support local paths, path start with http(s)://, and paths start with abfs(s):// # this is the default location so end users don't have to compile the runtime again. # feathr_runtime_location: wasbs://public@azurefeathrstorage.blob.core.windows.net/feathr-assembly-LATEST.jar - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" databricks: # workspace instance workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' @@ -93,7 +93,7 @@ spark_config: # Feathr Job location. Support local paths, path start with http(s)://, and paths start with dbfs:/ work_dir: 'dbfs:/feathr_getting_started' # this is the default location so end users don't have to compile the runtime again. - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml index afe923163..2df185fbe 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_purview.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' - workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db' + workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml index fb88972f7..ff347f59b 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_purview_rbac.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' - workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db' + workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml index 486eed1e4..215899c3a 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_sql.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' - workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db' + workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" online_store: redis: diff --git a/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml b/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml index 4ad7d35db..3b213b343 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_registry_sql_rbac.yaml @@ -25,13 +25,13 @@ spark_config: workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' executor_size: 'Small' executor_num: 1 - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" databricks: workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' - workspace_token_value: 'dapid8ddd83000dc2863763b7d47f0e8f3db' + workspace_token_value: '' config_template: {"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}} work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.8.0.jar" + feathr_runtime_location: "../../target/scala-2.12/feathr-assembly-0.9.0-rc2.jar" online_store: redis: diff --git a/feathr_project/test/unit/spark_provider/test_localspark_submission.py b/feathr_project/test/unit/spark_provider/test_localspark_submission.py new file mode 100644 index 000000000..9a9d7238b --- /dev/null +++ b/feathr_project/test/unit/spark_provider/test_localspark_submission.py @@ -0,0 +1,51 @@ +from typing import Dict +from unittest.mock import MagicMock + +import pytest +from pytest_mock import MockerFixture + +from feathr.spark_provider._localspark_submission import _FeathrLocalSparkJobLauncher + + +@pytest.fixture(scope="function") +def local_spark_job_launcher(tmp_path) -> _FeathrLocalSparkJobLauncher: + return _FeathrLocalSparkJobLauncher( + workspace_path=str(tmp_path), + debug_folder=str(tmp_path), + ) + + +def test__local_spark_job_launcher__submit_feathr_job( + mocker: MockerFixture, + local_spark_job_launcher: _FeathrLocalSparkJobLauncher, +): + # Mock necessary components + local_spark_job_launcher._init_args = MagicMock(return_value=[]) + mocked_proc = MagicMock() + mocked_proc.args = [] + mocked_proc.pid = 0 + + mocked_spark_proc = mocker.patch("feathr.spark_provider._localspark_submission.Popen", return_value=mocked_proc) + + local_spark_job_launcher.submit_feathr_job( + job_name="unit-test", + main_jar_path="", + main_class_name="", + ) + + # Assert if the mocked spark process has called once + mocked_spark_proc.assert_called_once() + + +@pytest.mark.parametrize( + "confs", [{}, {"spark.feathr.outputFormat": "parquet"}] +) +def test__local_spark_job_launcher__init_args( + local_spark_job_launcher: _FeathrLocalSparkJobLauncher, + confs: Dict[str, str], +): + spark_args = local_spark_job_launcher._init_args(job_name=None, confs=confs) + + # Assert if spark_args contains confs at the end + for k, v in confs.items(): + assert spark_args[-1] == f"{k}={v}" diff --git a/feathr_project/test/unit/udf/test_preprocessing_pyudf_manager.py b/feathr_project/test/unit/udf/test_preprocessing_pyudf_manager.py new file mode 100644 index 000000000..1daa87632 --- /dev/null +++ b/feathr_project/test/unit/udf/test_preprocessing_pyudf_manager.py @@ -0,0 +1,15 @@ +import pytest + +from feathr.udf._preprocessing_pyudf_manager import _PreprocessingPyudfManager + + +@pytest.mark.parametrize( + "fn_name, fn_str", + [ + ("fn_without_type_hint", "def fn_without_type_hint(a):\n return a + 10\n"), + ("fn_with_type_hint", "def fn_with_type_hint(a: int) -> int:\n return a + 10\n"), + ("fn_with_complex_type_hint", "def fn_with_complex_type_hint(a: Union[int, float]) -> Union[int, float]:\n return a + 10\n"), + ] +) +def test__parse_function_str_for_name(fn_name, fn_str): + assert fn_name == _PreprocessingPyudfManager._parse_function_str_for_name(fn_str) diff --git a/registry/access_control/rbac/access.py b/registry/access_control/rbac/access.py index a25646813..adee628c2 100644 --- a/registry/access_control/rbac/access.py +++ b/registry/access_control/rbac/access.py @@ -1,3 +1,4 @@ +from time import sleep from typing import Any, Union from uuid import UUID from fastapi import Depends, HTTPException, status @@ -23,6 +24,12 @@ def __init__(self, detail: Any = None) -> None: detail=detail, headers={"WWW-Authenticate": "Bearer"}) +class BadRequest(HTTPException): + def __init__(self, detail: Any = None) -> None: + super().__init__(status_code=status.HTTP_400_BAD_REQUEST, + detail=detail, headers={"WWW-Authenticate": "Bearer"}) + + def get_user(user: User = Depends(authorize)) -> User: return user @@ -72,13 +79,22 @@ def _get_project_name(id_or_name: Union[str, UUID]): _to_uuid(id_or_name) if id_or_name not in rbac.projects_ids: # refresh project id map if id not found - _get_projects_ids() + _get_projects_ids() + if id_or_name not in rbac.projects_ids: + # purview discovery-query api has latency, need retry to avoid new project not included issue. + # TODO: Update purview project-ids API to realtime one and remove below patch. + count = 0 + max = 5 + while id_or_name not in rbac.projects_ids and count < max: + sleep(0.5) + _get_projects_ids() + count += 1 return rbac.projects_ids[id_or_name] except KeyError: - raise RuntimeError(f"Project Id {id_or_name} not found in Registry {config.RBAC_REGISTRY_URL}") + raise BadRequest(f"Project Id {id_or_name} not found in Registry {config.RBAC_REGISTRY_URL}. Please check if the project exists or retry later.") except ValueError: + # It is a name pass - # It is a name return id_or_name @@ -88,4 +104,4 @@ def _get_projects_ids(): response = requests.get(url=f"{config.RBAC_REGISTRY_URL}/projects-ids").content.decode('utf-8') rbac.projects_ids = json.loads(response) except Exception as e: - raise RuntimeError(f"Failed to get projects ids from Registry {config.RBAC_REGISTRY_URL}, {e}") \ No newline at end of file + raise BadRequest(f"Failed to get projects ids from Registry {config.RBAC_REGISTRY_URL}, {e}") \ No newline at end of file diff --git a/registry/data-models/__init__.py b/registry/data-models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/registry/data-models/data-model-diagram.md b/registry/data-models/data-model-diagram.md new file mode 100644 index 000000000..e43ffa0af --- /dev/null +++ b/registry/data-models/data-model-diagram.md @@ -0,0 +1,61 @@ + +# Feathr Abstract backend Data Model Diagram + +This file defines abstract backend data models diagram for feature registry. +[Python Code](./models.py) + +```mermaid +classDiagram + Project "1" --> "n" FeatureName : contains + Project "1" --> "n" Anchor : contains + FeatureName "1" --> "n" Feature : contains + Anchor "1" --> "n" Feature : contains + Feature <|-- AnchorFeature : extends + Feature <|-- DerivedFeature: extends + Feature --> Transformation + Feature --> Transformation : contains + Source <|-- DataSource: extends + Source <|-- MultiFeatureSource: extends + MultiFeatureSource "1" --> "1..n" FeatureSource: contains + AnchorFeature --> DataSource : contains + DerivedFeature --> MultiFeatureSource: contains + + class Source{ + } + class DataSource{ + } + class FeatureSource{ + +FeatureNameId feature_name_id + } + class MultiFeatureSource{ + +List[FeatureSource] sources + } + class Feature{ + +FeatureId id + +FeatureNameId feature_namme_id + +Source source + +Transformation transformation + } + class AnchorFeature{ + +DataSource source + } + class DerivedFeature{ + +MultiFeatureSource source + } + class FeatureName{ + +FeatureNameId id + +ProjectId project_id + +List[FeatureId] feature_ids + } + class Project{ + +ProjectId id + +List[FeatureNameId] feature_name_ids + +List[AnchorId] anchor_ids + } + class Anchor{ + +AnchorId id + +ProjectId project_id + +DataSource source + +List[FeatureId] anchor_feature_ids + } +``` \ No newline at end of file diff --git a/registry/data-models/models.py b/registry/data-models/models.py new file mode 100644 index 000000000..c4ae31f68 --- /dev/null +++ b/registry/data-models/models.py @@ -0,0 +1,146 @@ +from pydantic import BaseModel +from typing import List + +""" +This file defines abstract backend data models for feature registry. +Backend data models will be used by backend API server to talk to feature registry backend. +Purpose of this is to decouple backend data models from API specific data models. +For each feature registry provider/implementation, they will extend this abstract +data models and backend API. +Diagram of the data models: ./data-model-diagram.md +""" + + +class FeatureId(BaseModel): + """ + Id for Feature, it's unique ID represents Feature. + Id can be a simple string, int or complex key. + """ + id: str # id of a feature + + +class FeatureNameId(BaseModel): + """ + Id for FeatureName, it's unique ID represents FeatureName. + Id can be a simple string, int or complex key. + """ + id: str # id of a FeatureName + + +class AnchorId(BaseModel): + """ + Id for Anchor, it's unique ID represents Anchor. + Id can be a simple string, int or complex key. + """ + id: str # id of a anchor + + +class ProjectId(BaseModel): + """ + Id for Project, it's unique ID represents Project. + Id can be a simple string, int or complex key. + """ + id: str # id of a project + + +class Source(BaseModel): + """ + Source of the feature. + It defines where the feature is extracted or derived from. + """ + pass + + +class DataSource(Source): + """ + Data source of the feature. + It defines the raw data source the feature is extracted from. + """ + pass + + +class FeatureSource(BaseModel): + """ + Represents a feature source for a derived feature. That is, it is a source 'FeatureName' which is used for + creating other derived features. + """ + input_feature_name_id: FeatureNameId # Input feature name Key + + +class MultiFeatureSource(Source): + """ + Feature sources of the feature. + It defines one to many features where the feature is derived from. + """ + sources: List[FeatureSource] # All source features which the feature is derived from + pass + + +class Transformation(BaseModel): + """ + The transformation of a Feature. + A transformation function represents the transformation logic to produce feature value from the source of FeatureAnchor + """ + pass + + +class Feature(BaseModel): + """ + Actual implementation of FeatureName. + An implementation defines where a feature is extracted from (Source) and how it is computed (Transformation). + The Source of a feature can be raw data sources and/or other features. + """ + id: FeatureId # Unique ID for Feature + feature_name_id: FeatureNameId # Id of the feature name that the feature belongs to + source: Source # Source can be either data source or feature source + transformation: Transformation # transformation logic to produce feature value + + +class AnchorFeature(Feature): + """ + Feature implementation of FeatureName which anchored to a data source. + """ + source: DataSource # Raw data source where the feature is extracted from + + +class DerivedFeature(Feature): + """ + Feature implementation that is derived from other FeatureNames. + """ + source: MultiFeatureSource # Source features where the feature is derived from + + +class FeatureName(BaseModel): + """ + Named Feature Interface that can be backed by multiple Feature implementations across + different environments accessing different sources (data lake access for batch training, + KV store access for online serving). Each FeatureName is defined by feature producer. + Feature consumers reference a feature by that name to access that feature data, + agnostic of runtime environment. Each FeatureName also encloses attributes that does not + change across implementations. + """ + id: FeatureNameId # unique ID for FeatureName, used to extract data for current FeatureName + project_id: ProjectId # ID of the project the FeatureName belongs to + feature_ids: List[FeatureId] # List of ids of feature that the FeatureName has + + +class Project(BaseModel): + """ + Group of FeatureNames. It can be a project the team is working on, + or a namespace which related FeatureNames have. + """ + id: ProjectId # Unique ID of the project. + feature_name_ids: List[FeatureNameId] # List of feature name ids that the project has + anchor_ids: List[AnchorId] # List of Anchor ids that the project has + + +class Anchor(BaseModel): + """ + Group of AnchorFeatures which anchored on same DataSource. + This is mainly used by feature producer gather information about DataSource + and FeatureImplementations associated with the DataSource. + """ + id: AnchorId # Unique ID for Anchor + project_id: ProjectId # ID of Project that the anchor belongs to + source: DataSource # data source of the Anchor + anchor_feature_ids: List[FeatureId] # List of anchor features that the anchor has diff --git a/registry/purview-registry/main.py b/registry/purview-registry/main.py index 92aa8dc49..5d38adf74 100644 --- a/registry/purview-registry/main.py +++ b/registry/purview-registry/main.py @@ -1,12 +1,11 @@ import os -import traceback from re import sub from typing import Optional from uuid import UUID from fastapi import APIRouter, FastAPI, HTTPException -from fastapi.responses import JSONResponse from starlette.middleware.cors import CORSMiddleware -from registry.purview_registry import PurviewRegistry, ConflictError +from registry import * +from registry.purview_registry import PurviewRegistry from registry.models import AnchorDef, AnchorFeatureDef, DerivedFeatureDef, EntityType, ProjectDef, SourceDef, to_snake rp = "/v1" @@ -44,48 +43,6 @@ def to_camel(s): allow_headers=["*"], ) -def exc_to_content(e: Exception) -> dict: - content={"message": str(e)} - if os.environ.get("REGISTRY_DEBUGGING"): - content["traceback"] = "".join(traceback.TracebackException.from_exception(e).format()) - return content - -@app.exception_handler(ConflictError) -async def conflict_error_handler(_, exc: ConflictError): - return JSONResponse( - status_code=409, - content=exc_to_content(exc), - ) - - -@app.exception_handler(ValueError) -async def value_error_handler(_, exc: ValueError): - return JSONResponse( - status_code=400, - content=exc_to_content(exc), - ) - -@app.exception_handler(TypeError) -async def type_error_handler(_, exc: ValueError): - return JSONResponse( - status_code=400, - content=exc_to_content(exc), - ) - - -@app.exception_handler(KeyError) -async def key_error_handler(_, exc: KeyError): - return JSONResponse( - status_code=404, - content=exc_to_content(exc), - ) - -@app.exception_handler(IndexError) -async def index_error_handler(_, exc: IndexError): - return JSONResponse( - status_code=404, - content=exc_to_content(exc), - ) @router.get("/projects",tags=["Project"]) def get_projects() -> list[str]: diff --git a/registry/purview-registry/registry/purview_registry.py b/registry/purview-registry/registry/purview_registry.py index 06d7bd8d1..15a650167 100644 --- a/registry/purview-registry/registry/purview_registry.py +++ b/registry/purview-registry/registry/purview_registry.py @@ -1,15 +1,20 @@ import copy +from http.client import CONFLICT, HTTPException import itertools -from typing import Optional, Tuple, Union +from typing import Any, Optional, Tuple, Union +from urllib.error import HTTPError from uuid import UUID +from registry.models import to_snake +from pyapacheatlas.core.util import AtlasException + from azure.identity import DefaultAzureCredential from loguru import logger from pyapacheatlas.auth.azcredential import AzCredentialWrapper from pyapacheatlas.core import (AtlasEntity, AtlasProcess, PurviewClient) from pyapacheatlas.core.typedef import (AtlasAttributeDef,Cardinality,EntityTypeDef) -from pyapacheatlas.core.util import GuidTracker, AtlasException +from pyapacheatlas.core.util import GuidTracker from pyhocon import ConfigFactory from registry.interface import Registry @@ -18,13 +23,12 @@ Label_BelongsTo = "BELONGSTO" Label_Consumes = "CONSUMES" Label_Produces = "PRODUCES" +TYPEDEF_DERIVED_FEATURE="feathr_derived_feature_v1" +TYPEDEF_ANCHOR_FEATURE="feathr_anchor_feature_v1" + TYPEDEF_ARRAY_ANCHOR=f"array" TYPEDEF_ARRAY_DERIVED_FEATURE=f"array" TYPEDEF_ARRAY_ANCHOR_FEATURE=f"array" - -class ConflictError(Exception): - pass - class PurviewRegistry(Registry): def __init__(self,azure_purview_name: str, registry_delimiter: str = "__", credential=None,register_types = True): self.registry_delimiter = registry_delimiter @@ -572,20 +576,46 @@ def _upload_entity_batch(self, entity_batch:list[AtlasEntity]): # setting lastModifiedTS ==0 will ensure this, if another entity with ts>=1 exist # upload function will fail with 412 Precondition fail. for entity in entity_batch: - entity.lastModifiedTS="0" - try: - results = self.purview_client.upload_entities( - batch=entity) - if results: - dict = {x.guid: x for x in entity_batch} - for k, v in results['guidAssignments'].items(): - dict[k].guid = v + self._upload_single_entity(entity) + + def _upload_single_entity(self, entity:AtlasEntity): + try: + """ + Try to find existing entity/process first, if found, return the existing entity's GUID + """ + id = self.get_entity_id(entity.qualifiedName) + response = self.purview_client.get_entity(id)['entities'][0] + j = entity.to_json() + if j["typeName"] == response["typeName"]: + if j["typeName"] == "Process": + if response["attributes"]["qualifiedName"] != j["attributes"]["qualifiedName"]: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) else: - raise RuntimeError("Feature registration failed.", results) - except AtlasException as e: - if "PreConditionCheckFailed" in e.args[0]: - raise ConflictError(f"Entity {entity.guid}, {entity.typeName} -- {entity.qualifiedName} already exists in Purview. Please use a new name.") + if "type" in response['attributes'] and response["typeName"] in (TYPEDEF_ANCHOR_FEATURE, TYPEDEF_DERIVED_FEATURE): + conf = ConfigFactory.parse_string(response['attributes']['type']) + response['attributes']['type'] = dict(conf) + keys = set([to_snake(key) for key in j["attributes"].keys()]) - set(["qualified_name"]) + keys.add("qualifiedName") + for k in keys: + if response["attributes"][k] != j["attributes"][k]: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + entity.guid = response["guid"] + return + else: + raise RuntimeError("The requested entity %s conflicts with the existing entity in PurView" % j["attributes"]["qualifiedName"]) + except AtlasException as e: + pass + entity.lastModifiedTS="0" + results = self.purview_client.upload_entities( + batch=entity) + if results: + d = {x.guid: x for x in [entity]} + for k, v in results['guidAssignments'].items(): + d[k].guid = v + else: + raise RuntimeError("Feature registration failed.", results) + def _generate_fully_qualified_name(self, segments): return self.registry_delimiter.join(segments) diff --git a/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala b/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala index 2e38e4d04..185c9d2d6 100644 --- a/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala +++ b/src/main/scala/com/linkedin/feathr/common/AnchorExtractor.scala @@ -1,7 +1,5 @@ package com.linkedin.feathr.common -import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema - /** * Provides feature values based on some "raw" data element * @@ -39,12 +37,14 @@ trait AnchorExtractor[T] extends AnchorExtractorBase[T] with SparkRowExtractor { * @param datum input row * @return list of feature keys */ - def getKeyFromRow(datum: GenericRowWithSchema): Seq[String] = getKey(datum.asInstanceOf[T]) + def getKeyFromRow(datum: Any): Seq[String] = getKey(datum.asInstanceOf[T]) /** * Get the feature value from the row * @param datum input row * @return A map of feature name to feature value */ - def getFeaturesFromRow(datum: GenericRowWithSchema): Map[String, FeatureValue] = getFeatures(datum.asInstanceOf[T]) + def getFeaturesFromRow(datum: Any): Map[String, FeatureValue] = getFeatures(datum.asInstanceOf[T]) + + override def toString: String = getClass.getSimpleName } diff --git a/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala b/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala new file mode 100644 index 000000000..7051a308c --- /dev/null +++ b/src/main/scala/com/linkedin/feathr/common/CanConvertToAvroRDD.scala @@ -0,0 +1,20 @@ +package com.linkedin.feathr.common + +import org.apache.avro.generic.IndexedRecord +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.DataFrame + +/** + * If an AnchorExtractor only works on a Avro record, it should extends + * this trait, and use convertToAvroRdd to do a one-time batch conversion of DataFrame to RDD of their choice. + * convertToAvroRdd will be called by Feathr engine before calling getKeyFromRow() and getFeaturesFromRow() in AnchorExtractor. + */ +trait CanConvertToAvroRDD { + + /** + * One time batch converting the input data source into a RDD[IndexedRecord] for feature extraction later + * @param df input data source + * @return batch preprocessed dataframe, as RDD[IndexedRecord] + */ + def convertToAvroRdd(df: DataFrame) : RDD[IndexedRecord] +} diff --git a/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala b/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala index 04e715e8c..ad088ac0a 100644 --- a/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala +++ b/src/main/scala/com/linkedin/feathr/common/SparkRowExtractor.scala @@ -1,7 +1,5 @@ package com.linkedin.feathr.common -import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema - /** * An extractor trait that provides APIs to transform a Spark GenericRowWithSchema into feature values */ @@ -12,12 +10,12 @@ trait SparkRowExtractor { * @param datum input row * @return list of feature keys */ - def getKeyFromRow(datum: GenericRowWithSchema): Seq[String] + def getKeyFromRow(datum: Any): Seq[String] /** * Get the feature value from the row * @param datum input row * @return A map of feature name to feature value */ - def getFeaturesFromRow(datum: GenericRowWithSchema): Map[String, FeatureValue] + def getFeaturesFromRow(datum: Any): Map[String, FeatureValue] } \ No newline at end of file diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala b/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala index 59f5bfbe7..edb2e2c06 100644 --- a/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala +++ b/src/main/scala/com/linkedin/feathr/offline/anchored/anchorExtractor/SimpleConfigurableAnchorExtractor.scala @@ -10,7 +10,6 @@ import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext import com.linkedin.feathr.offline.mvel.{MvelContext, MvelUtils} import com.linkedin.feathr.offline.util.FeatureValueTypeValidator import org.apache.log4j.Logger -import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.types._ import org.mvel2.MVEL @@ -66,7 +65,7 @@ private[offline] class SimpleConfigurableAnchorExtractor( @JsonProperty("key") k * @param datum input row * @return list of feature keys */ - override def getKeyFromRow(datum: GenericRowWithSchema): Seq[String] = { + override def getKeyFromRow(datum: Any): Seq[String] = { getKey(datum.asInstanceOf[Any]) } @@ -107,7 +106,7 @@ private[offline] class SimpleConfigurableAnchorExtractor( @JsonProperty("key") k * @param row input row * @return A map of feature name to feature value */ - override def getFeaturesFromRow(row: GenericRowWithSchema) = { + override def getFeaturesFromRow(row: Any) = { getFeatures(row.asInstanceOf[Any]) } @@ -147,7 +146,7 @@ private[offline] class SimpleConfigurableAnchorExtractor( @JsonProperty("key") k featureTypeConfigs(featureRefStr) } val featureValue = offline.FeatureValue.fromTypeConfig(value, featureTypeConfig) - FeatureValueTypeValidator.validate(featureValue, featureTypeConfigs(featureRefStr)) + FeatureValueTypeValidator.validate(featureRefStr, featureValue, featureTypeConfigs(featureRefStr) ) (featureRefStr, featureValue) } diff --git a/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala b/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala index 209ac89e1..bf5108e8b 100644 --- a/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala +++ b/src/main/scala/com/linkedin/feathr/offline/anchored/keyExtractor/MVELSourceKeyExtractor.scala @@ -43,7 +43,7 @@ private[feathr] class MVELSourceKeyExtractor(val anchorExtractorV1: AnchorExtrac .toDF() } - def getKey(datum: GenericRowWithSchema): Seq[String] = { + def getKey(datum: Any): Seq[String] = { anchorExtractorV1.getKeyFromRow(datum) } @@ -55,7 +55,7 @@ private[feathr] class MVELSourceKeyExtractor(val anchorExtractorV1: AnchorExtrac */ override def getKeyColumnNames(datum: Option[Any]): Seq[String] = { if (datum.isDefined) { - val size = getKey(datum.get.asInstanceOf[GenericRowWithSchema]).size + val size = getKey(datum.get).size (1 to size).map(JOIN_KEY_PREFIX + _) } else { // return empty join key to signal empty dataset @@ -86,5 +86,6 @@ private[feathr] class MVELSourceKeyExtractor(val anchorExtractorV1: AnchorExtrac // this helps to reduce the number of joins // to the observation data // The default toString does not work, because toString of each object have different values - override def toString: String = getClass.getSimpleName + " with keyExprs:" + keyExprs.mkString(" key:") + override def toString: String = getClass.getSimpleName + " with keyExprs:" + keyExprs.mkString(" key:") + + "anchorExtractor:" + anchorExtractorV1.toString } diff --git a/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala b/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala index aee596097..aadeaf50c 100644 --- a/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala +++ b/src/main/scala/com/linkedin/feathr/offline/config/FeathrConfigLoader.scala @@ -327,7 +327,7 @@ private[offline] class AnchorLoader extends JsonDeserializer[FeatureAnchor] { case Some(tType) => offline.FeatureValue.fromTypeConfig(rawValue, tType) case None => offline.FeatureValue(rawValue, featureType, key) } - FeatureValueTypeValidator.validate(featureValue, featureTypeConfig) + FeatureValueTypeValidator.validate(featureValue, featureTypeConfig, key) (key, featureValue) } .toMap diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala b/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala index ff16ebe18..59dd8ea8e 100644 --- a/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala +++ b/src/main/scala/com/linkedin/feathr/offline/derived/DerivedFeatureEvaluator.scala @@ -1,19 +1,19 @@ package com.linkedin.feathr.offline.derived -import com.linkedin.feathr.{common, offline} -import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig} import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} -import com.linkedin.feathr.offline.{ErasedEntityTaggedFeature, FeatureDataFrame} +import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureTypeConfig} import com.linkedin.feathr.offline.client.DataFrameColName import com.linkedin.feathr.offline.client.plugins.{FeathrUdfPluginContext, FeatureDerivationFunctionAdaptor} -import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SeqJoinDerivationFunction} -import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinAsDerivation, SparkUdfDerivation} +import com.linkedin.feathr.offline.derived.functions.{MvelFeatureDerivationFunction, SQLFeatureDerivationFunction, SeqJoinDerivationFunction} +import com.linkedin.feathr.offline.derived.strategies._ import com.linkedin.feathr.offline.join.algorithms.{SequentialJoinConditionBuilder, SparkJoinWithJoinCondition} import com.linkedin.feathr.offline.logical.FeatureGroups import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext -import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils import com.linkedin.feathr.offline.source.accessor.DataPathHandler +import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils +import com.linkedin.feathr.offline.{ErasedEntityTaggedFeature, FeatureDataFrame} import com.linkedin.feathr.sparkcommon.FeatureDerivationFunctionSpark +import com.linkedin.feathr.{common, offline} import org.apache.log4j.Logger import org.apache.spark.sql.{DataFrame, SparkSession} @@ -45,6 +45,9 @@ private[offline] class DerivedFeatureEvaluator(derivationStrategies: DerivationS case h: FeatureDerivationFunctionSpark => val resultDF = derivationStrategies.customDerivationSparkStrategy(keyTag, keyTagList, contextDF, derivedFeature, h, mvelContext) convertFeatureColumnToQuinceFds(producedFeatureColName, derivedFeature, resultDF) + case s: SQLFeatureDerivationFunction => + val resultDF = derivationStrategies.sqlDerivationSparkStrategy(keyTag, keyTagList, contextDF, derivedFeature, s, mvelContext) + convertFeatureColumnToQuinceFds(producedFeatureColName, derivedFeature, resultDF) case x: FeatureDerivationFunction => // We should do the FDS conversion inside the rowBasedDerivationStrategy here. The result of rowBasedDerivationStrategy // can be NTV FeatureValue or TensorData-based Feature. NTV FeatureValue has fixed FDS schema. However, TensorData @@ -118,8 +121,8 @@ private[offline] object DerivedFeatureEvaluator { val defaultStrategies = strategies.DerivationStrategies( new SparkUdfDerivation(), new RowBasedDerivation(featureGroups.allTypeConfigs, mvelContext), - new SequentialJoinAsDerivation(ss, featureGroups, SparkJoinWithJoinCondition(SequentialJoinConditionBuilder), dataPathHandlers) - ) + new SequentialJoinAsDerivation(ss, featureGroups, SparkJoinWithJoinCondition(SequentialJoinConditionBuilder), dataPathHandlers), + new SqlDerivationSpark()) new DerivedFeatureEvaluator(defaultStrategies, mvelContext) } diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala b/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala index e54d68f59..13fbec9c7 100644 --- a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala +++ b/src/main/scala/com/linkedin/feathr/offline/derived/strategies/DerivationStrategies.scala @@ -1,8 +1,8 @@ package com.linkedin.feathr.offline.derived.strategies import com.linkedin.feathr.common.{FeatureDerivationFunction, FeatureDerivationFunctionBase} -import com.linkedin.feathr.offline.derived.functions.SeqJoinDerivationFunction import com.linkedin.feathr.offline.derived.DerivedFeature +import com.linkedin.feathr.offline.derived.functions.{SQLFeatureDerivationFunction, SeqJoinDerivationFunction} import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext import com.linkedin.feathr.sparkcommon.FeatureDerivationFunctionSpark import org.apache.spark.sql.DataFrame @@ -41,10 +41,17 @@ private[offline] trait RowBasedDerivationStrategy extends DerivationStrategy[Fea */ private[offline] trait SequentialJoinDerivationStrategy extends DerivationStrategy[SeqJoinDerivationFunction] +/** + * Implementation should define how a SQL-expression based derivation is evaluated. + */ +private[offline] trait SqlDerivationSparkStrategy extends DerivationStrategy[SQLFeatureDerivationFunction] + /** * This case class holds the implementations of supported strategies. */ private[offline] case class DerivationStrategies( customDerivationSparkStrategy: SparkUdfDerivationStrategy, rowBasedDerivationStrategy: RowBasedDerivationStrategy, - sequentialJoinDerivationStrategy: SequentialJoinDerivationStrategy) + sequentialJoinDerivationStrategy: SequentialJoinDerivationStrategy, + sqlDerivationSparkStrategy: SqlDerivationSparkStrategy) { +} diff --git a/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala b/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala new file mode 100644 index 000000000..c7b44c1cf --- /dev/null +++ b/src/main/scala/com/linkedin/feathr/offline/derived/strategies/SqlDerivationSpark.scala @@ -0,0 +1,118 @@ +package com.linkedin.feathr.offline.derived.strategies + +import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrFeatureTransformationException} +import com.linkedin.feathr.offline.client.DataFrameColName +import com.linkedin.feathr.offline.derived.DerivedFeature +import com.linkedin.feathr.offline.derived.functions.SQLFeatureDerivationFunction +import com.linkedin.feathr.offline.job.FeatureTransformation.FEATURE_NAME_PREFIX +import com.linkedin.feathr.offline.mvel.plugins.FeathrExpressionExecutionContext +import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.{DataFrame, SparkSession} + +import scala.collection.JavaConverters._ + +/** + * This class executes SQL-expression based derived feature. + */ +class SqlDerivationSpark extends SqlDerivationSparkStrategy { + + + /** + * Rewrite sqlExpression for a derived feature, e.g, replace the feature name/argument name with Frame internal dataframe column name + * @param deriveFeature derived feature definition + * @param keyTag list of tags represented by integer + * @param keyTagId2StringMap Map from the tag integer id to the string tag + * @param asIsFeatureNames features names that does not to be rewritten, i.e. passthrough features, as they do not have key tags + * @return Rewritten SQL expression + */ + private[offline] def rewriteDerivedFeatureExpression( + deriveFeature: DerivedFeature, + keyTag: Seq[Int], + keyTagId2StringMap: Seq[String], + asIsFeatureNames: Set[String]): String = { + if (!deriveFeature.derivation.isInstanceOf[SQLFeatureDerivationFunction]) { + throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_ERROR, "Should not rewrite derived feature expression for non-SQLDerivedFeatures") + } + val sqlDerivation = deriveFeature.derivation.asInstanceOf[SQLFeatureDerivationFunction] + val deriveExpr = sqlDerivation.getExpression() + val parameterNames: Seq[String] = sqlDerivation.getParameterNames().getOrElse(Seq[String]()) + val consumedFeatureNames = deriveFeature.consumedFeatureNames.zipWithIndex.map { + case (consumeFeatureName, index) => + // begin of string, or other char except number and alphabet + // val featureStartPattern = """(^|[^a-zA-Z0-9])""" + // end of string, or other char except number and alphabet + // val featureEndPattern = """($|[^a-zA-Z0-9])""" + val namePattern = if (parameterNames.isEmpty) consumeFeatureName.getFeatureName else parameterNames(index) + // getBinding.map(keyTag.get) resolves the call tags + val newName = + if (!asIsFeatureNames.contains(FEATURE_NAME_PREFIX + consumeFeatureName.getFeatureName) + // Feature generation code path does not create columns with tags. + // The check ensures we do not run into IndexOutOfBoundsException when keyTag & keyTagId2StringMap are empty. + && keyTag.nonEmpty + && keyTagId2StringMap.nonEmpty) { + DataFrameColName.genFeatureColumnName( + consumeFeatureName.getFeatureName, + Some(consumeFeatureName.getBinding.asScala.map(keyTag(_)).map(keyTagId2StringMap))) + } else { + DataFrameColName.genFeatureColumnName(consumeFeatureName.getFeatureName) + } + (namePattern, newName) + }.toMap + + // replace all feature name to column names + // featureName is consist of numAlphabetic + val ss: SparkSession = SparkSession.builder().getOrCreate() + val dependencyFeatures = ss.sessionState.sqlParser.parseExpression(deriveExpr).references.map(_.name).toSeq + // \w is [a-zA-Z0-9_], not inclusion of _ and exclusion of -, as - is ambiguous, e.g, a-b could be a feature name or feature a minus feature b + val rewrittenExpr = dependencyFeatures.foldLeft(deriveExpr)((acc, ca) => { + // in scala \W does not work as ^\w + // "a+B+1".replaceAll("([^\w])B([^\w])", "$1abc$2" = A+abc+1 + // "a+B".replaceAll("([^\w])B$", "$1abc" = a+abc + // "B+1".replaceAll("^B([^\w])", "abc$1" = abc+1 + // "B".replaceAll("^B$", "abc" = abc + val newVal = consumedFeatureNames.getOrElse(ca, ca) + val patterns = Seq("([^\\w])" + ca + "([^\\w])", "([^\\w])" + ca + "$", "^" + ca + "([^\\w])", "^" + ca + "$") + val replacements = Seq("$1" + newVal + "$2", "$1" + newVal, newVal + "$1", newVal) + val replacedExpr = patterns + .zip(replacements) + .toMap + .foldLeft(acc)((orig, pairs) => { + orig.replaceAll(pairs._1, pairs._2) + }) + replacedExpr + }) + rewrittenExpr + } + + /** + * Apply the derivation strategy. + * + * @param keyTags keyTags for the derived feature. + * @param keyTagList integer keyTag to string keyTag map. + * @param df input DataFrame. + * @param derivedFeature Derived feature metadata. + * @param derivationFunction Derivation function to evaluate the derived feature + * @return output DataFrame with derived feature. + */ + override def apply(keyTags: Seq[Int], + keyTagList: Seq[String], + df: DataFrame, + derivedFeature: DerivedFeature, + derivationFunction: SQLFeatureDerivationFunction, + mvelContext: Option[FeathrExpressionExecutionContext]): DataFrame = { + // sql expression based derived feature needs rewrite, e.g, replace the feature names with feature column names in the dataframe + // Passthrough fields do not need rewrite as they do not have tags. + val passthroughFieldNames = df.schema.fields.map(f => + if (f.name.startsWith(FEATURE_NAME_PREFIX)) { + f.name + } else { + FEATURE_NAME_PREFIX + f.name + } + ).toSet + val rewrittenExpr = rewriteDerivedFeatureExpression(derivedFeature, keyTags, keyTagList, passthroughFieldNames) + val tags = Some(keyTags.map(keyTagList).toList) + val featureColumnName = DataFrameColName.genFeatureColumnName(derivedFeature.producedFeatureNames.head, tags) + df.withColumn(featureColumnName, expr(rewrittenExpr)) + } + +} diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala b/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala index 310c3931e..57f4def55 100644 --- a/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala +++ b/src/main/scala/com/linkedin/feathr/offline/generation/DataFrameFeatureGenerator.scala @@ -5,7 +5,7 @@ import com.linkedin.feathr.common.{Header, JoiningFeatureParams, TaggedFeatureNa import com.linkedin.feathr.offline import com.linkedin.feathr.offline.anchored.feature.FeatureAnchorWithSource.{getDefaultValues, getFeatureTypes} import com.linkedin.feathr.offline.derived.functions.SeqJoinDerivationFunction -import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinDerivationStrategy, SparkUdfDerivation} +import com.linkedin.feathr.offline.derived.strategies.{DerivationStrategies, RowBasedDerivation, SequentialJoinDerivationStrategy, SparkUdfDerivation, SqlDerivationSpark} import com.linkedin.feathr.offline.derived.{DerivedFeature, DerivedFeatureEvaluator} import com.linkedin.feathr.offline.evaluator.DerivedFeatureGenStage import com.linkedin.feathr.offline.job.{FeatureGenSpec, FeatureTransformation} @@ -133,5 +133,7 @@ private[offline] class DataFrameFeatureGenerator(logicalPlan: MultiStageJoinPlan ErrorLabel.FEATHR_ERROR, s"Feature Generation does not support Sequential Join features : ${derivedFeature.producedFeatureNames.head}") } - }), mvelContext) + }, + new SqlDerivationSpark() + ), mvelContext) } diff --git a/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala b/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala index 99436b93c..126128323 100644 --- a/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala +++ b/src/main/scala/com/linkedin/feathr/offline/generation/StreamingFeatureGenerator.scala @@ -6,7 +6,7 @@ import com.linkedin.feathr.common.JoiningFeatureParams import com.linkedin.feathr.offline.config.location.KafkaEndpoint import com.linkedin.feathr.offline.generation.outputProcessor.PushToRedisOutputProcessor.TABLE_PARAM_CONFIG_NAME import com.linkedin.feathr.offline.generation.outputProcessor.RedisOutputUtils -import com.linkedin.feathr.offline.job.FeatureTransformation.getFeatureJoinKey +import com.linkedin.feathr.offline.job.FeatureTransformation.getFeatureKeyColumnNames import com.linkedin.feathr.offline.job.{FeatureGenSpec, FeatureTransformation} import com.linkedin.feathr.offline.logical.FeatureGroups import com.linkedin.feathr.offline.source.accessor.DataPathHandler @@ -111,7 +111,7 @@ class StreamingFeatureGenerator(dataPathHandlers: List[DataPathHandler]) { // Apply feature transformation val transformedResult = DataFrameBasedSqlEvaluator.transform(anchor.featureAnchor.extractor.asInstanceOf[SimpleAnchorExtractorSpark], withKeyColumnDF, featureNamePrefixPairs, anchor.featureAnchor.featureTypeConfigs) - val outputJoinKeyColumnNames = getFeatureJoinKey(keyExtractor, withKeyColumnDF) + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(keyExtractor, withKeyColumnDF) val selectedColumns = outputJoinKeyColumnNames ++ anchor.selectedFeatures.filter(keyTaggedFeatures.map(_.featureName).contains(_)) val cleanedDF = transformedResult.df.select(selectedColumns.head, selectedColumns.tail:_*) val keyColumnNames = FeatureTransformation.getStandardizedKeyNames(outputJoinKeyColumnNames.size) diff --git a/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala b/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala index 94de8e645..7b106572b 100644 --- a/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala +++ b/src/main/scala/com/linkedin/feathr/offline/job/FeatureTransformation.scala @@ -1,7 +1,9 @@ package com.linkedin.feathr.offline.job -import com.linkedin.feathr.common._ import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException, FeathrFeatureTransformationException} +import com.linkedin.feathr.common.tensor.TensorData +import com.linkedin.feathr.common.types.FeatureType +import com.linkedin.feathr.common.{AnchorExtractorBase, _} import com.linkedin.feathr.offline.anchored.anchorExtractor.{SQLConfigurableAnchorExtractor, SimpleConfigurableAnchorExtractor, TimeWindowConfigurableAnchorExtractor} import com.linkedin.feathr.offline.anchored.feature.{FeatureAnchor, FeatureAnchorWithSource} import com.linkedin.feathr.offline.anchored.keyExtractor.MVELSourceKeyExtractor @@ -22,6 +24,7 @@ import com.linkedin.feathr.offline.{FeatureDataFrame, JoinKeys} import com.linkedin.feathr.sparkcommon.{SimpleAnchorExtractorSpark, SourceKeyExtractor} import com.linkedin.feathr.swj.aggregate.AggregationType import com.linkedin.feathr.{common, offline} +import org.apache.avro.generic.IndexedRecord import org.apache.log4j.Logger import org.apache.spark.rdd.RDD import org.apache.spark.sql.functions._ @@ -41,6 +44,16 @@ import scala.concurrent.{Await, ExecutionContext, Future} */ private[offline] case class AnchorFeatureGroups(anchorsWithSameSource: Seq[FeatureAnchorWithSource], requestedFeatures: Seq[String]) +/** + * Context info needed in feature transformation + * @param featureAnchorWithSource feature annchor with its source + * @param featureNamePrefixPairs map of feature name to its prefix + * @param transformer transformer of anchor + */ +private[offline] case class TransformInfo(featureAnchorWithSource: FeatureAnchorWithSource, + featureNamePrefixPairs: Seq[(FeatureName, FeatureName)], + transformer: AnchorExtractorBase[IndexedRecord]) + /** * Represent the transformed result of an anchor extractor after evaluating its features * @param featureNameAndPrefixPairs pairs of feature name and feature name prefix @@ -75,7 +88,27 @@ private[offline] object FeatureTransformation { // feature name, column prefix type FeatureNameAndColumnPrefix = (String, String) - def getFeatureJoinKey(sourceKeyExtractor: SourceKeyExtractor, withKeyColumnDF: DataFrame, featureExtractor: Option[AnyRef] = None): Seq[String] = { + /** + * Extract feature key column names from the input feature RDD using the sourceKeyExtractor. + * @param sourceKeyExtractor key extractor that knows what are the key column in a feature RDD. + * @param withKeyColumnRDD RDD that contains the key columns. + * @return feature key column names + */ + def getFeatureKeyColumnNamesRdd(sourceKeyExtractor: SourceKeyExtractor, withKeyColumnRDD: RDD[_]): Seq[String] = { + if (withKeyColumnRDD.isEmpty) { + sourceKeyExtractor.getKeyColumnNames(None) + } else { + sourceKeyExtractor.getKeyColumnNames(Some(withKeyColumnRDD.first())) + } + } + + /** + * Extract feature key column names from the input feature DataFrame using the sourceKeyExtractor. + * @param sourceKeyExtractor key extractor that knows what are the key column in a feature RDD. + * @param withKeyColumnDF DataFrame that contains the key columns. + * @return feature key column names + */ + def getFeatureKeyColumnNames(sourceKeyExtractor: SourceKeyExtractor, withKeyColumnDF: DataFrame): Seq[String] = { if (withKeyColumnDF.head(1).isEmpty) { sourceKeyExtractor.getKeyColumnNames(None) } else { @@ -306,7 +339,8 @@ private[offline] object FeatureTransformation { } val withKeyColumnDF = keyExtractor.appendKeyColumns(sourceDF) - val outputJoinKeyColumnNames = getFeatureJoinKey(keyExtractor, withKeyColumnDF, Some(anchorFeatureGroup.anchorsWithSameSource.head.featureAnchor.extractor)) + + val outputJoinKeyColumnNames = getFeatureKeyColumnNames(keyExtractor, withKeyColumnDF) val filteredFactData = applyBloomFilter((keyExtractor, withKeyColumnDF), bloomFilter) // 1. apply all transformations on the dataframe in sequential order @@ -457,10 +491,21 @@ private[offline] object FeatureTransformation { val keyExtractor = anchorsWithSameSource.head._1.featureAnchor.sourceKeyExtractor val featureAnchorWithSource = anchorsWithSameSource.keys.toSeq val selectedFeatures = anchorsWithSameSource.flatMap(_._2.featureNames).toSeq - - val sourceDF = featureGroupingFactors.source - val transformedResults: Seq[KeyedTransformedResult] = transformMultiAnchorsOnSingleDataFrame(sourceDF, + val isAvroRddBasedExtractor = featureAnchorWithSource + .map(_.featureAnchor.extractor) + .filter(extractor => extractor.isInstanceOf[CanConvertToAvroRDD] + ).nonEmpty + val transformedResults: Seq[KeyedTransformedResult] = if (isAvroRddBasedExtractor) { + // If there are features are defined using AVRO record based extractor, run RDD based feature transformation + val sourceAccessor = featureGroupingFactors.source + val sourceRdd = sourceAccessor.asInstanceOf[NonTimeBasedDataSourceAccessor].get() + val featureTypeConfigs = featureAnchorWithSource.flatMap(featureAnchor => featureAnchor.featureAnchor.featureTypeConfigs).toMap + Seq(transformFeaturesOnAvroRecord(sourceRdd, keyExtractor, featureAnchorWithSource, bloomFilter, selectedFeatures, featureTypeConfigs)) + } else { + val sourceDF = featureGroupingFactors.source + transformFeaturesOnDataFrameRow(sourceDF, keyExtractor, featureAnchorWithSource, bloomFilter, selectedFeatures, incrementalAggContext, mvelContext) + } val res = transformedResults .map { transformedResultWithKey => @@ -673,6 +718,204 @@ private[offline] object FeatureTransformation { } } + + /** + * Apply a bloomfilter to a RDD + * + * @param keyExtractor key extractor to extract the key values from the RDD + * @param rdd RDD to filter + * @param bloomFilter bloomfilter used to filter out unwanted row in the RDD based on key columns + * @return filtered RDD + */ + + private def applyBloomFilterRdd(keyExtractor: SourceKeyExtractor, rdd: RDD[IndexedRecord], bloomFilter: Option[BloomFilter]): RDD[IndexedRecord] = { + bloomFilter match { + case None => + // no bloom filter, use data as it + rdd + case Some(filter) => + // get the list of join key columns or expression + keyExtractor match { + case extractor: MVELSourceKeyExtractor => + // get the list of join key columns or expression + val keyColumnsList = if (rdd.isEmpty) { + extractor.getKeyColumnNames(None) + } else { + extractor.getKeyColumnNames(Some(rdd.first)) + } + if (!keyColumnsList.isEmpty) { + val filtered = rdd.filter { record: Any => + val keyVals = extractor.getKey(record) + // if key is not in observation, skip it + if (keyVals != null && keyVals.count(_ == null) == 0) { + filter.mightContainString(SourceUtils.generateFilterKeyString(keyVals)) + } else { + false + } + } + filtered + } else { + // expand feature for seq join does not have right key, so we allow empty here + rdd + } + case _ => throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_USER_ERROR, "No source key extractor found") + } + } + } + + /** + * Transform features defined in a group of anchors based on same source + * This is for the AVRO record based extractors + * + * @param rdd source that requested features are defined on + * @param keyExtractor key extractor to apply on source rdd + * @param featureAnchorWithSources feature anchors defined on source rdd to be evaluated + * @param bloomFilter bloomfilter to apply on source rdd + * @param requestedFeatureNames requested features + * @param featureTypeConfigs user specified feature types + * @return TransformedResultWithKey The output feature DataFrame conforms to FDS format + */ + private def transformFeaturesOnAvroRecord(df: DataFrame, + keyExtractor: SourceKeyExtractor, + featureAnchorWithSources: Seq[FeatureAnchorWithSource], + bloomFilter: Option[BloomFilter], + requestedFeatureNames: Seq[FeatureName], + featureTypeConfigs: Map[String, FeatureTypeConfig] = Map()): KeyedTransformedResult = { + if (!keyExtractor.isInstanceOf[MVELSourceKeyExtractor]) { + throw new FeathrException(ErrorLabel.FEATHR_ERROR, s"Error processing requested Feature :${requestedFeatureNames}. " + + s"Key extractor ${keyExtractor} must extends MVELSourceKeyExtractor.") + } + val extractor = keyExtractor.asInstanceOf[MVELSourceKeyExtractor] + if (!extractor.anchorExtractorV1.isInstanceOf[CanConvertToAvroRDD]) { + throw new FeathrException(ErrorLabel.FEATHR_ERROR, s"Error processing requested Feature :${requestedFeatureNames}. " + + s"isLowLevelRddExtractor() should return true and convertToAvroRdd should be implemented.") + } + val rdd = extractor.anchorExtractorV1.asInstanceOf[CanConvertToAvroRDD].convertToAvroRdd(df) + val filteredFactData = applyBloomFilterRdd(keyExtractor, rdd, bloomFilter) + + // Build a sequence of 3-tuple of (FeatureAnchorWithSource, featureNamePrefixPairs, AnchorExtractorBase) + val transformInfo = featureAnchorWithSources map { featureAnchorWithSource => + val extractor = featureAnchorWithSource.featureAnchor.extractor + extractor match { + case transformer: AnchorExtractorBase[IndexedRecord] => + // We no longer need prefix for the simplicity of the implementation, instead if there's a feature name + // and source data field clash, we will throw exception and ask user to rename the feature. + val featureNamePrefix = "" + val featureNames = featureAnchorWithSource.selectedFeatures.filter(requestedFeatureNames.contains) + val featureNamePrefixPairs = featureNames.map((_, featureNamePrefix)) + TransformInfo(featureAnchorWithSource, featureNamePrefixPairs, transformer) + + case _ => + throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_USER_ERROR, s"Unsupported transformer $extractor for features: $requestedFeatureNames") + } + } + + // to avoid name conflict between feature names and the raw data field names + val sourceKeyExtractors = transformInfo.map(_.featureAnchorWithSource.featureAnchor.sourceKeyExtractor) + assert(sourceKeyExtractors.map(_.toString).distinct.size == 1) + + val transformers = transformInfo map (_.transformer) + + /* + * Transform the given RDD by applying extractors to each row to create an RDD[Row] where each Row + * represents keys and feature values + */ + val spark = SparkSession.builder().getOrCreate() + val userProvidedFeatureTypes = transformInfo.flatMap(_.featureAnchorWithSource.featureAnchor.getFeatureTypes.getOrElse(Map.empty[String, FeatureTypes])).toMap + val FeatureTypeInferenceContext(featureTypeAccumulators) = + FeatureTransformation.getTypeInferenceContext(spark, userProvidedFeatureTypes, requestedFeatureNames) + val transformedRdd = filteredFactData map { record => + val (keys, featureValuesWithType) = transformAvroRecord(requestedFeatureNames, sourceKeyExtractors, transformers, record, featureTypeConfigs) + requestedFeatureNames.zip(featureValuesWithType).foreach { + case (featureRef, (_, featureType)) => + if (featureTypeAccumulators(featureRef).isZero && featureType != null) { + // This is lazy evaluated + featureTypeAccumulators(featureRef).add(FeatureTypes.valueOf(featureType.getBasicType.toString)) + } + } + // Create a row by merging a row created from keys and a row created from term-vectors/tensors + Row.merge(Row.fromSeq(keys), Row.fromSeq(featureValuesWithType.map(_._1))) + } + + // Create a DataFrame from the above obtained RDD + val keyNames = getFeatureKeyColumnNamesRdd(sourceKeyExtractors.head, filteredFactData) + val (outputSchema, inferredFeatureTypeConfigs) = { + val allFeatureTypeConfigs = featureAnchorWithSources.flatMap(featureAnchorWithSource => featureAnchorWithSource.featureAnchor.featureTypeConfigs).toMap + val inferredFeatureTypes = inferFeatureTypes(featureTypeAccumulators, transformedRdd, requestedFeatureNames) + val inferredFeatureTypeConfigs = inferredFeatureTypes.map(x => x._1 -> new FeatureTypeConfig(x._2)) + val mergedFeatureTypeConfig = inferredFeatureTypeConfigs ++ allFeatureTypeConfigs + val colPrefix = "" + val featureTensorTypeInfo = getFDSSchemaFields(requestedFeatureNames, mergedFeatureTypeConfig, colPrefix) + val structFields = keyNames.foldRight(List.empty[StructField]) { + case (colName, acc) => + StructField(colName, StringType) :: acc + } + val outputSchema = StructType(StructType(structFields ++ featureTensorTypeInfo)) + (outputSchema, mergedFeatureTypeConfig) + } + val transformedDF = spark.createDataFrame(transformedRdd, outputSchema) + + val featureFormat = FeatureColumnFormat.FDS_TENSOR + val featureColumnFormats = requestedFeatureNames.map(name => name -> featureFormat).toMap + val transformedInfo = TransformedResult(transformInfo.flatMap(_.featureNamePrefixPairs), transformedDF, featureColumnFormats, inferredFeatureTypeConfigs) + KeyedTransformedResult(keyNames, transformedInfo) + } + + /** + * Apply a keyExtractor and feature transformer on a Record to extractor feature values. + * @param requestedFeatureNames requested feature names in the output. Extractors may produce more features than requested. + * @param sourceKeyExtractors extractor to extract the key from the record + * @param transformers transform to produce the feature value from the record + * @param record avro record to work on + * @param featureTypeConfigs user defined feature types + * @return tuple of (feature join key, sequence of (feature value, feature type) in the order of requestedFeatureNames) + */ + private def transformAvroRecord( + requestedFeatureNames: Seq[FeatureName], + sourceKeyExtractors: Seq[SourceKeyExtractor], + transformers: Seq[AnchorExtractorBase[IndexedRecord]], + record: IndexedRecord, + featureTypeConfigs: Map[String, FeatureTypeConfig] = Map()): (Seq[String], Seq[(Any, FeatureType)]) = { + val keys = sourceKeyExtractors.head match { + case mvelSourceKeyExtractor: MVELSourceKeyExtractor => mvelSourceKeyExtractor.getKey(record) + case _ => throw new FeathrFeatureTransformationException(ErrorLabel.FEATHR_USER_ERROR, s"${sourceKeyExtractors.head} is not a valid extractor on RDD") + } + + /* + * For the given row, apply all extractors to extract feature values. If requested as tensors, each feature value + * contains a tensor else a term-vector. + */ + val features = transformers map { + case extractor: AnchorExtractor[IndexedRecord] => + val features = extractor.getFeatures(record) + FeatureValueTypeValidator.validate(features, featureTypeConfigs) + features + case extractor => + throw new FeathrFeatureTransformationException( + ErrorLabel.FEATHR_USER_ERROR, + s"Invalid extractor $extractor for features:" + + s"$requestedFeatureNames requested as tensors") + } reduce (_ ++ _) + if (logger.isTraceEnabled) { + logger.trace(s"Extracted features: $features") + } + + /* + * Retain feature values for only the requested features, and represent each feature value as + * a tensor, as specified. + */ + val featureValuesWithType = requestedFeatureNames map { name => + features.get(name) map { + case featureValue => + val tensorData: TensorData = featureValue.getAsTensorData() + val featureType: FeatureType = featureValue.getFeatureType() + val row = FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData) + (row, featureType) + } getOrElse ((null, null)) // return null if no feature value present + } + (keys, featureValuesWithType) + } + /** * Helper function to be used by groupFeatures. Given a collection of feature anchors which also contains information about grouping * criteria and extractor type per feature anchor, returns a map of FeatureGroupingCriteria to @@ -851,7 +1094,7 @@ private[offline] object FeatureTransformation { * others use direct aggregation * */ - private def transformMultiAnchorsOnSingleDataFrame( + private def transformFeaturesOnDataFrameRow( source: DataSourceAccessor, keyExtractor: SourceKeyExtractor, anchorsWithSameSource: Seq[FeatureAnchorWithSource], @@ -878,7 +1121,7 @@ private[offline] object FeatureTransformation { val incrAggCtx = incrementalAggContext.get val preAggDFs = incrAggCtx.previousSnapshotMap.collect { case (featureName, df) if requestedFeatures.exists(df.columns.contains) => df }.toSeq.distinct // join each previous aggregation dataframe sequentially - val groupKeys = getFeatureJoinKey(keyExtractor, preAggDFs.head) + val groupKeys = getFeatureKeyColumnNames(keyExtractor, preAggDFs.head) val keyColumnNames = getStandardizedKeyNames(groupKeys.size) val firstPreAgg = preAggDFs.head val joinedPreAggDFs = preAggDFs diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala b/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala index d242372bf..cc6cba1c7 100644 --- a/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala +++ b/src/main/scala/com/linkedin/feathr/offline/transformation/DataFrameBasedRowEvaluator.scala @@ -74,19 +74,20 @@ private[offline] object DataFrameBasedRowEvaluator { val featureTypes = featureTypeConfigs.mapValues(_.getFeatureType) val FeatureTypeInferenceContext(featureTypeAccumulators) = FeatureTransformation.getTypeInferenceContext(spark, featureTypes, featureRefStrs) + val transformedRdd = inputDF.rdd.map(row => { - // in some cases, the input dataframe row here only have Row and does not have schema attached, - // while MVEL only works with GenericRowWithSchema, create it manually - val rowWithSchema = if (row.isInstanceOf[GenericRowWithSchema]) { - row.asInstanceOf[GenericRowWithSchema] - } else { - new GenericRowWithSchema(row.toSeq.toArray, inputSchema) - } - if (rowExtractor.isInstanceOf[SimpleConfigurableAnchorExtractor]) { - rowExtractor.asInstanceOf[SimpleConfigurableAnchorExtractor].mvelContext = mvelContext - } - val result = rowExtractor.getFeaturesFromRow(rowWithSchema) - val featureValues = featureRefStrs map { + // in some cases, the input dataframe row here only have Row and does not have schema attached, + // while MVEL only works with GenericRowWithSchema, create it manually + val rowWithSchema = if (row.isInstanceOf[GenericRowWithSchema]) { + row.asInstanceOf[GenericRowWithSchema] + } else { + new GenericRowWithSchema(row.toSeq.toArray, inputSchema) + } + if (rowExtractor.isInstanceOf[SimpleConfigurableAnchorExtractor]) { + rowExtractor.asInstanceOf[SimpleConfigurableAnchorExtractor].mvelContext = mvelContext + } + val result = rowExtractor.getFeaturesFromRow(rowWithSchema) + val featureValues = featureRefStrs map { featureRef => if (result.contains(featureRef)) { val featureValue = result(featureRef) @@ -95,7 +96,7 @@ private[offline] object DataFrameBasedRowEvaluator { featureTypeAccumulators(featureRef).add(FeatureTypes.valueOf(rowFeatureType.toString)) } val tensorData: TensorData = featureValue.getAsTensorData() - FeaturizedDatasetUtils.tensorToDataFrameRow(tensorData) + FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData) } else null } Row.merge(row, Row.fromSeq(featureValues)) diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala b/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala index 366967cc2..bf5d70c75 100644 --- a/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala +++ b/src/main/scala/com/linkedin/feathr/offline/transformation/DefaultValueSubstituter.scala @@ -112,7 +112,7 @@ private[offline] object DataFrameDefaultValueSubstituter extends DataFrameDefaul // For tensor default, since we don't have type, so we need to use expr to construct the default column val schema = field.dataType val tensorData = defaultFeatureValue.getAsTensorData - val ts = FeaturizedDatasetUtils.tensorToDataFrameRow(tensorData) + val ts = FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData, Some(schema)) val fdsTensorDefaultUDF = getFDSTensorDefaultUDF(schema, ts) ss.udf.register("tz_udf", fdsTensorDefaultUDF) expr(s"tz_udf($featureColumnName)") diff --git a/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala b/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala index 824f48fe3..25d96af11 100644 --- a/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala +++ b/src/main/scala/com/linkedin/feathr/offline/transformation/FDSConversionUtils.scala @@ -2,14 +2,13 @@ package com.linkedin.feathr.offline.transformation import com.linkedin.feathr.common.exception.{ErrorLabel, FeathrException} import com.linkedin.feathr.common.tensor.TensorData - -import java.util import com.linkedin.feathr.common.util.CoercionUtils import com.linkedin.feathr.offline.util.FeaturizedDatasetUtils import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.types._ +import java.util import scala.collection.JavaConverters._ import scala.collection.convert.Wrappers.JMapWrapper import scala.collection.mutable @@ -37,7 +36,7 @@ private[offline] object FDSConversionUtils { // convert the "raw" input data into a FDS column a specific dataType rawFeatureValue match { case tensorData: TensorData => - FeaturizedDatasetUtils.tensorToDataFrameRow(tensorData, Some(targetDataType)) + FeaturizedDatasetUtils.tensorToFDSDataFrameRow(tensorData, Some(targetDataType)) case _ => targetDataType match { // Scalar tensor @@ -253,7 +252,13 @@ private[offline] object FDSConversionUtils { case values: util.ArrayList[Any] => values.asScala.toArray case values: mutable.WrappedArray[Any] => - values.asInstanceOf[mutable.WrappedArray[Any]].toArray + if (values.nonEmpty && values(0).isInstanceOf[GenericRowWithSchema]) { + // Assuming the result is returned by SWA feature with groupBy, hence keeping only the + // feature value as an array and dropping the index info. + values.asInstanceOf[mutable.WrappedArray[GenericRowWithSchema]].map(v => v.get(v.size - 1)).toArray + } else { + values.toArray + } case values: List[Any] => values.toArray case mapValues: Map[Integer, Any] => diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala b/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala index ee06f3acd..aec0b1aea 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala +++ b/src/main/scala/com/linkedin/feathr/offline/util/FeatureValueTypeValidator.scala @@ -16,7 +16,7 @@ private[offline] object FeatureValueTypeValidator { features.foreach { case (key, value) => featureTypeConfigs.get(key).foreach( - featureTypeConfig => FeatureValueTypeValidator.validate(value, featureTypeConfig)) + featureTypeConfig => FeatureValueTypeValidator.validate(key, value, featureTypeConfig)) } } @@ -27,9 +27,9 @@ private[offline] object FeatureValueTypeValidator { * @param featureValue value extracted from data * @param featureTypeConfig user-defined config, optional */ - def validate(featureValue: FeatureValue, featureTypeConfig: Option[FeatureTypeConfig]): Unit = { + def validate(featureValue: FeatureValue, featureTypeConfig: Option[FeatureTypeConfig], featureName: String): Unit = { featureTypeConfig match { - case Some(f) => validate(featureValue, f) + case Some(f) => validate(featureName, featureValue, f) case None => } } @@ -41,31 +41,31 @@ private[offline] object FeatureValueTypeValidator { * @param featureValue value extracted from data * @param featureTypeConfig user-defined config */ - def validate(featureValue: FeatureValue, featureTypeConfig: FeatureTypeConfig): Unit = { + def validate(featureName: String, featureValue: FeatureValue, featureTypeConfig: FeatureTypeConfig): Unit = { val configFeatureTypes = featureTypeConfig.getFeatureType val valueBasicType = featureValue.getFeatureType.getBasicType if (configFeatureTypes != FeatureTypes.UNSPECIFIED) { if (valueBasicType != FeatureType.BasicType.TENSOR || configFeatureTypes != FeatureTypes.TENSOR) { if (configFeatureTypes != FeatureTypes.valueOf(valueBasicType.name)) { - throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The FeatureValue type: " + valueBasicType - + " is not consistent with the type specified in the Feathr config: ." + configFeatureTypes); + throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The FeatureValue type of : " + featureName + + " is " + valueBasicType + ", which is not consistent with the type specified in the Feathr config: ." + configFeatureTypes); } } else if (featureTypeConfig.getTensorType != null) { val configTensorType = featureTypeConfig.getTensorType val valueTensorType = featureValue.getAsTypedTensor.getType if (configTensorType.getValueType != null && configTensorType.getValueType != valueTensorType.getValueType) { - throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor value type: " + valueTensorType - + " is not consistent with the type specified in the Feathr config: ." + configTensorType); + throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor value type of :" + featureName + + " is " + valueTensorType + ", which is not consistent with the type specified in the Feathr config: ." + configTensorType); } if (configTensorType.getTensorCategory != null && configTensorType.getTensorCategory != valueTensorType.getTensorCategory) { - throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor category type: " + valueTensorType - + " is not consistent with the type specified in the Feathr config: ." + configTensorType); + throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor category type of : " + featureName + " is " + + valueTensorType + ", which is not consistent with the type specified in the Feathr config: ." + configTensorType); } if (configTensorType.getDimensionTypes != null && configTensorType.getDimensionTypes != valueTensorType.getDimensionTypes) { - throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor dimension type: " + valueTensorType - + " is not consistent with the type specified in the Feathr config: ." + configTensorType); + throw new FeathrException(ErrorLabel.FEATHR_USER_ERROR, "The tensor dimension type of : " + featureName + " is " + + valueTensorType + ", which is not consistent with the type specified in the Feathr config: ." + configTensorType); } } } diff --git a/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala b/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala index d672cf5f5..534881f7a 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala +++ b/src/main/scala/com/linkedin/feathr/offline/util/FeaturizedDatasetUtils.scala @@ -157,7 +157,7 @@ private[offline] object FeaturizedDatasetUtils { * @return the Quince-FDS struct or primitive */ - def tensorToDataFrameRow(tensor: TensorData, targetDataType: Option[DataType] = None): Any = { + def tensorToFDSDataFrameRow(tensor: TensorData, targetDataType: Option[DataType] = None): Any = { tensor match { case null => null case _ => diff --git a/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala b/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala index 1604e174c..1673786f5 100644 --- a/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala +++ b/src/main/scala/com/linkedin/feathr/offline/util/SourceUtils.scala @@ -655,11 +655,7 @@ private[offline] object SourceUtils { ss.read.format("csv").option("header", "true").option("delimiter", csvDelimiterOption).load(inputData.inputPath) } case _ => { - if (ss.sparkContext.isLocal){ - getLocalDF(ss, inputData.inputPath, dataLoaderHandlers) - } else { - loadAsDataFrame(ss, SimplePath(inputData.inputPath),dataLoaderHandlers) - } + loadAsDataFrame(ss, SimplePath(inputData.inputPath),dataLoaderHandlers) } } } diff --git a/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala b/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala index 061b42598..3735c0f9f 100644 --- a/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala +++ b/src/test/scala/com/linkedin/feathr/offline/AnchoredFeaturesIntegTest.scala @@ -58,6 +58,16 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { | type: "DENSE_VECTOR" | default: [7,8,9] | } + | ee2: { + | def: "c" + | type: { + | type: TENSOR + | tensorCategory: DENSE + | dimensionType: [INT] + | valType: FLOAT + | } + | default: [] + | } | ff: { | def: "c" | default: [6,7] @@ -155,7 +165,7 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { */ @Test def testSingleKeyJoinWithDifferentFeatureTypes(): Unit = { - val selectedColumns = Seq("x", "aa", "bb", "cc", "dd", "ee", "ff", "multiply_a_b", "categorical_b") // , "z") + val selectedColumns = Seq("x", "aa", "bb", "cc", "dd", "ee", "ee2", "ff", "multiply_a_b", "categorical_b") // , "z") val featureJoinConf = s""" | @@ -186,6 +196,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { null, // ee mutable.WrappedArray.make(Array(7.0f, 8.0f, 9.0f)), + // ee2 + mutable.WrappedArray.empty, // ff mutable.WrappedArray.make(Array(6.0f, 7.0f)), // multiply_a_b @@ -207,6 +219,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // ee mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), + // ee2 + mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // ff mutable.WrappedArray.make(Array(1.0f, 2.0f, 3.0f)), // multiply_a_b @@ -228,6 +242,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // ee mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), + // ee2 + mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // ff mutable.WrappedArray.make(Array(4.0f, 5.0f, 6.0f)), // multiply_a_b @@ -246,6 +262,7 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { StructField("cc", FloatType, true), StructField("dd", ArrayType(FloatType, true), true), StructField("ee", ArrayType(FloatType, false), true), + StructField("ee2", ArrayType(FloatType, false), true), StructField("ff", ArrayType(FloatType, false), true), StructField( "multiply_a_b", @@ -467,7 +484,16 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { | |derivations: { | f_trip_time_distance: { - | definition: "f_trip_distance * f_trip_time_duration" + | definition: "f_trip_distance * f_trip_time_duration" + | type: NUMERIC + | } + | f_trip_time_distance_sql: { + | key: [trip] + | inputs: { + | trip_distance: { key: [trip], feature: f_trip_distance } + | trip_time_duration: { key: [trip], feature: f_trip_time_duration } + | } + | definition.sqlExpr: "trip_distance * trip_time_duration" | type: NUMERIC | } |} @@ -497,7 +523,8 @@ class AnchoredFeaturesIntegTest extends FeathrIntegTest { |featureList: [ | { | key: DOLocationID - | featureList: [f_location_avg_fare, f_trip_time_distance, f_trip_distance, f_trip_time_duration, f_is_long_trip_distance, f_day_of_week] + | featureList: [f_location_avg_fare, f_trip_time_distance, f_trip_distance, + | f_trip_time_duration, f_is_long_trip_distance, f_day_of_week, f_trip_time_distance_sql] | } |] """.stripMargin diff --git a/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala b/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala new file mode 100644 index 000000000..94e92e06d --- /dev/null +++ b/src/test/scala/com/linkedin/feathr/offline/DerivationsIntegTest.scala @@ -0,0 +1,146 @@ +package com.linkedin.feathr.offline + +import com.linkedin.feathr.offline.util.FeathrTestUtils.assertDataFrameApproximatelyEquals +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ +import org.testng.annotations.Test + +class DerivationsIntegTest extends FeathrIntegTest { + + /** + * Test multi-key derived feature and multi-tagged feature. + * This test covers the following:- + * -> sql based custom extractor + */ + @Test + def testMultiKeyDerivedFeatureDFWithSQL: Unit = { + val df = runLocalFeatureJoinForTest( + joinConfigAsString = """ + | features: [ { + | key: ["concat('',viewer)", viewee] + | featureList: [ "foo_square_distance_sql"] + | } , + | { + | key: [viewee, viewer] + | featureList: [ "foo_square_distance_sql"] + | }, + | { + | key: [viewee, viewer] + | featureList: [ "square_fooFeature_sql"] + | } + | ] + """.stripMargin, + featureDefAsString = """ + | anchors: { + | anchor1: { + | source: anchorAndDerivations/derivations/anchor6-source.csv + | key.sqlExpr: [sourceId, destId] + | features: { + | fooFeature: { + | def.sqlExpr: cast(source as int) + | type: NUMERIC + | } + | } + | } + | } + | derivations: { + | + | square_fooFeature_sql: { + | key: [m1, m2] + | inputs: { + | a: { key: [m1, m2], feature: fooFeature } + | } + | definition.sqlExpr: "a * a" + | } + | foo_square_distance_sql: { + | key: [m1, m2] + | inputs: { + | a1: { key: [m1, m2], feature: square_fooFeature_sql } + | a2: { key: [m2, m1], feature: square_fooFeature_sql } + | } + | definition.sqlExpr: "a1 - a2" + | } + | } + """.stripMargin, + observationDataPath = "anchorAndDerivations/derivations/test2-observations.csv") + + val expectedDf = ss.createDataFrame( + ss.sparkContext.parallelize( + Seq( + Row( + // viewer + "1", + // viewee + "3", + // label + "1.0", + // square_fooFeature_sql + 4.0f, + // viewee_viewer__foo_square_distance_sql + -21.0f, + // concat____viewer__viewee__foo_square_distance_sql + 21.0f), + Row( + // viewer + "2", + // viewee + "1", + // label + "-1.0", + // square_fooFeature_sql + 9.0f, + // viewee_viewer__foo_square_distance_sql + -27.0f, + // concat____viewer__viewee__foo_square_distance_sql + 27.0f), + Row( + // viewer + "3", + // viewee + "6", + // label + "1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null), + Row( + // viewer + "3", + // viewee + "5", + // label + "-1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null), + Row( + // viewer + "5", + // viewee + "10", + // label + "1.0", + // square_fooFeature_sql + null, + // viewee_viewer__foo_square_distance_sql + null, + // concat____viewer__viewee__foo_square_distance_sql + null))), + StructType( + List( + StructField("viewer", StringType, true), + StructField("viewee", StringType, true), + StructField("label", StringType, true), + StructField("square_fooFeature_sql", FloatType, true), + StructField("viewee_viewer__foo_square_distance_sql", FloatType, true), + StructField("concat____viewer__viewee__foo_square_distance_sql", FloatType, true)))) + def cmpFunc(row: Row): String = if (row.get(0) != null) row.get(0).toString else "null" + assertDataFrameApproximatelyEquals(df.data, expectedDf, cmpFunc) + } +} diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala b/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala index 3ab94e616..f3b75024e 100644 --- a/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala +++ b/src/test/scala/com/linkedin/feathr/offline/util/TestFDSConversionUtil.scala @@ -3,18 +3,17 @@ package com.linkedin.feathr.offline.util import com.linkedin.feathr.common.TensorUtils import com.linkedin.feathr.common.tensor.{TensorType, Tensors} import com.linkedin.feathr.common.types.PrimitiveType - -import java.util -import java.util.Collections import com.linkedin.feathr.offline.AssertFeatureUtils import com.linkedin.feathr.offline.transformation.FDSConversionUtils import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.expressions.GenericRow +import org.apache.spark.sql.catalyst.expressions.{GenericRow, GenericRowWithSchema} import org.apache.spark.sql.types._ import org.scalatest.testng.TestNGSuite import org.testng.Assert.{assertEquals, assertTrue} import org.testng.annotations.{DataProvider, Test} +import java.util +import java.util.Collections import scala.collection.mutable class TestFDSConversionUtil extends TestNGSuite { @@ -141,10 +140,18 @@ class TestFDSConversionUtil extends TestNGSuite { @DataProvider def dataForTestConvertRawValueTo1DFDSDenseTensorRowTz(): Array[Array[Any]] = { + val eleType = StructType( + StructField("group", IntegerType, false) :: + StructField("value", IntegerType, false) :: Nil + ) + val row1 = new GenericRowWithSchema(Array(1, 3), eleType) + val row2 = new GenericRowWithSchema(Array(2, 4), eleType) Array( Array(mutable.WrappedArray.make(Array(2.0f, 6.0f)), util.Arrays.asList(2.0f, 6.0f).toArray), Array(Array(1.1).toList, util.Arrays.asList(1.1).toArray), - Array(Map("a" -> 1.1), util.Arrays.asList(1.1).toArray) + Array(Map("a" -> 1.1), util.Arrays.asList(1.1).toArray), + // Simulate raw value return by SWA feature with groupBy + Array(mutable.WrappedArray.make(Array(row1, row2)), util.Arrays.asList(3, 4).toArray) ) } @Test(dataProvider = "dataForTestConvertRawValueTo1DFDSDenseTensorRowTz") diff --git a/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala b/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala index 1e9bae9b7..bda25b1cc 100644 --- a/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala +++ b/src/test/scala/com/linkedin/feathr/offline/util/TestFeatureValueTypeValidator.scala @@ -45,7 +45,7 @@ class TestFeatureValueTypeValidator extends TestFeathr { new FeatureValue(value, valueFeatureType.asInstanceOf[FeatureTypes]); } val featureTypeConfig = new FeatureTypeConfig(configFeatureTypes.asInstanceOf[FeatureTypes], configTensorType.asInstanceOf[TensorType], null) - FeatureValueTypeValidator.validate(featureValue, featureTypeConfig) + FeatureValueTypeValidator.validate("", featureValue, featureTypeConfig) } @DataProvider(name = "failTestCases") @@ -75,7 +75,7 @@ class TestFeatureValueTypeValidator extends TestFeathr { new FeatureValue(value, valueFeatureType.asInstanceOf[FeatureTypes]); } val featureTypeConfig = new FeatureTypeConfig(configFeatureTypes.asInstanceOf[FeatureTypes], configTensorType.asInstanceOf[TensorType], null) - FeatureValueTypeValidator.validate(featureValue, featureTypeConfig) + FeatureValueTypeValidator.validate("", featureValue, featureTypeConfig) } diff --git a/ui/.editorconfig b/ui/.editorconfig new file mode 100644 index 000000000..b5e435a15 --- /dev/null +++ b/ui/.editorconfig @@ -0,0 +1,10 @@ +# http://editorconfig.org +root = true + +[*] +charset=utf-8 +end_of_line=lf +insert_final_newline=false +indent_style=space +indent_size=2 + diff --git a/ui/.eslintrc b/ui/.eslintrc index 2a16ad386..c271bfa24 100644 --- a/ui/.eslintrc +++ b/ui/.eslintrc @@ -4,7 +4,15 @@ "es6": true, "node": true }, - "plugins": ["@typescript-eslint/eslint-plugin"], + "plugins": ["react", "@typescript-eslint/eslint-plugin", "prettier"], + "settings": { + "import/resolver": { + "node": { + "extensions": [".tsx", ".ts", ".jsx", ".js", ".json"] + }, + "typescript": {} + } + }, "extends": [ // https://github.com/eslint/eslint/blob/main/conf/eslint-recommended.js "eslint:recommended", @@ -12,7 +20,8 @@ "react-app", // https://reactjs.org/docs/hooks-rules.html "plugin:react-hooks/recommended", - "prettier" + "plugin:prettier/recommended", + "plugin:json/recommended" ], "parser": "@typescript-eslint/parser", "parserOptions": { @@ -20,7 +29,22 @@ "sourceType": "module" }, "rules": { - "dot-notation": "error" + "dot-notation": "error", + "import/extensions": [ + "error", + "ignorePackages", + { + "ts": "never", + "tsx": "never", + "js": "never", + "jsx": "never" + } + ], + "import/no-extraneous-dependencies": ["error", { "devDependencies": true }], + "import/prefer-default-export": "off", + "import/no-unresolved": "error", + "import/no-dynamic-require": "off", + "import/no-mutable-exports": "warn" }, "overrides": [ { diff --git a/ui/.vscode/settings.json b/ui/.vscode/settings.json index 24fe97ae7..5fffcb522 100644 --- a/ui/.vscode/settings.json +++ b/ui/.vscode/settings.json @@ -4,5 +4,15 @@ }, "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true, - "eslint.workingDirectories": [{ "mode": "auto" }] + "eslint.workingDirectories": [ + { + "mode": "auto" + } + ], + "[css]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + } } diff --git a/ui/craco.config.js b/ui/craco.config.js new file mode 100644 index 000000000..e44884899 --- /dev/null +++ b/ui/craco.config.js @@ -0,0 +1,79 @@ +const path = require("path"); + +const { loaderByName } = require("@craco/craco"); +const CracoLessPlugin = require("craco-less"); + +const webpack = require("webpack"); + +const packageJson = require("./package.json"); + +const resolve = (dir) => path.resolve(__dirname, dir); + +const currentTime = new Date(); + +module.exports = { + babel: { + plugins: [ + [ + "import", + { + libraryName: "antd", + libraryDirectory: "es", + style: true, + }, + ], + ], + }, + webpack: { + alias: { + "@": resolve("src"), + }, + configure: (webpackConfig, { env, paths }) => { + const index = webpackConfig.plugins.findIndex( + (itme) => itme instanceof webpack.DefinePlugin + ); + + if (index > -1) { + const definePlugin = webpackConfig.plugins[index]; + webpackConfig.plugins.splice( + index, + 1, + new webpack.DefinePlugin({ + "process.env": { + ...definePlugin.definitions["process.env"], + FEATHR_VERSION: JSON.stringify(packageJson.version), + FEATHR_GENERATED_TIME: JSON.stringify(currentTime.toISOString()), + }, + }) + ); + } + + return webpackConfig; + }, + }, + plugins: [ + { + plugin: CracoLessPlugin, + options: { + lessLoaderOptions: { + lessOptions: { + modifyVars: {}, + javascriptEnabled: true, + }, + }, + modifyLessModuleRule(lessModuleRule, context) { + // Configure the file suffix + lessModuleRule.test = /\.module\.less$/; + + // Configure the generated local ident name. + const cssLoader = lessModuleRule.use.find(loaderByName("css-loader")); + cssLoader.options.modules = { + localIdentName: "[local]_[hash:base64:5]", + }; + + return lessModuleRule; + }, + }, + }, + ], +}; diff --git a/ui/package-lock.json b/ui/package-lock.json index b3a0d27d8..480dfdc62 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,18 +8,23 @@ "name": "feathr-ui", "version": "0.1.0", "dependencies": { + "@ant-design/icons": "^4.7.0", "@azure/msal-browser": "^2.24.0", "@azure/msal-react": "^1.4.0", - "antd": "^4.20.2", + "antd": "^4.23.6", "axios": "^0.27.2", + "classnames": "^2.3.2", "dagre": "^0.8.5", + "dayjs": "^1.11.5", "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^9.7.4", "react-query": "^3.38.0", + "react-resizable": "^3.0.4", "react-router-dom": "^6.3.0" }, "devDependencies": { + "@craco/craco": "^7.0.0-alpha.8", "@testing-library/jest-dom": "^5.16.3", "@testing-library/react": "^12.1.4", "@testing-library/user-event": "^13.5.0", @@ -28,17 +33,25 @@ "@types/node": "^16.11.26", "@types/react": "^17.0.43", "@types/react-dom": "^17.0.14", + "@types/react-resizable": "^3.0.3", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", + "babel-plugin-import": "^1.13.5", + "craco-less": "^2.1.0-alpha.0", "eslint": "^8.20.0", "eslint-config-prettier": "^8.5.0", + "eslint-import-resolver-typescript": "^3.5.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.1", "lint-staged": "^13.0.3", "prettier": "2.7.1", "react-scripts": "5.0.0", "typescript": "^4.6.3", - "web-vitals": "^2.1.4" + "web-vitals": "^2.1.4", + "webpack": "^5.72.0" } }, "node_modules/@ampproject/remapping": { @@ -62,7 +75,8 @@ }, "node_modules/@ant-design/icons": { "version": "4.7.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz", + "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==", "dependencies": { "@ant-design/colors": "^6.0.0", "@ant-design/icons-svg": "^4.2.1", @@ -83,14 +97,15 @@ "license": "MIT" }, "node_modules/@ant-design/react-slick": { - "version": "0.28.4", - "license": "MIT", + "version": "0.29.2", + "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz", + "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==", "dependencies": { "@babel/runtime": "^7.10.4", "classnames": "^2.2.5", "json2mq": "^0.2.0", "lodash": "^4.17.21", - "resize-observer-polyfill": "^1.5.0" + "resize-observer-polyfill": "^1.5.1" }, "peerDependencies": { "react": ">=16.9.0" @@ -1935,10 +1950,11 @@ } }, "node_modules/@babel/runtime": { - "version": "7.17.9", - "license": "MIT", + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.0.tgz", + "integrity": "sha512-NDYdls71fTXoU8TZHfbBWg7DiZfNzClcKui/+kyi6ppD2L1qnWW3VV6CjtaBXSUGGhiTWJ6ereOIkUvenif66Q==", "dependencies": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.10" }, "engines": { "node": ">=6.9.0" @@ -2006,6 +2022,43 @@ "dev": true, "license": "MIT" }, + "node_modules/@craco/craco": { + "version": "7.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@craco/craco/-/craco-7.0.0-alpha.8.tgz", + "integrity": "sha512-IN3/ldPaktGflPu342cg7n8LYa2c3x9H2XzngUkDzTjro25ig1GyVcUdnG1U0X6wrRTF9K1AxZ5su9jLbdyFUw==", + "dev": true, + "dependencies": { + "autoprefixer": "^10.4.12", + "cosmiconfig": "^7.0.1", + "cosmiconfig-typescript-loader": "^4.1.1", + "cross-spawn": "^7.0.3", + "lodash": "^4.17.21", + "semver": "^7.3.7", + "webpack-merge": "^5.8.0" + }, + "bin": { + "craco": "dist/bin/craco.js" + }, + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "react-scripts": "^5.0.0" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "peer": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@csstools/normalize.css": { "version": "12.0.0", "dev": true, @@ -2768,6 +2821,32 @@ "node": ">= 8" } }, + "node_modules/@pkgr/utils": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.3.1.tgz", + "integrity": "sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.3", + "is-glob": "^4.0.3", + "open": "^8.4.0", + "picocolors": "^1.0.0", + "tiny-glob": "^0.2.9", + "tslib": "^2.4.0" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/@pkgr/utils/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, "node_modules/@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.5", "dev": true, @@ -3245,6 +3324,34 @@ "node": ">=10.13.0" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "peer": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true, + "peer": true + }, "node_modules/@types/aria-query": { "version": "4.2.2", "dev": true, @@ -3505,6 +3612,15 @@ "redux": "^4.0.0" } }, + "node_modules/@types/react-resizable": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/react-resizable/-/react-resizable-3.0.3.tgz", + "integrity": "sha512-W/QsUOZoXBAIBQNhNm95A5ohoaiUA874lWQytO2UP9dOjp5JHO9+a0cwYNabea7sA12ZDJnGVUFZxcNaNksAWA==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/resolve": { "version": "1.17.1", "dev": true, @@ -4464,52 +4580,53 @@ } }, "node_modules/antd": { - "version": "4.20.2", - "license": "MIT", + "version": "4.23.6", + "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz", + "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==", "dependencies": { "@ant-design/colors": "^6.0.0", "@ant-design/icons": "^4.7.0", - "@ant-design/react-slick": "~0.28.1", - "@babel/runtime": "^7.12.5", + "@ant-design/react-slick": "~0.29.1", + "@babel/runtime": "^7.18.3", "@ctrl/tinycolor": "^3.4.0", "classnames": "^2.2.6", "copy-to-clipboard": "^3.2.0", "lodash": "^4.17.21", "memoize-one": "^6.0.0", "moment": "^2.29.2", - "rc-cascader": "~3.5.0", + "rc-cascader": "~3.7.0", "rc-checkbox": "~2.3.0", - "rc-collapse": "~3.1.0", - "rc-dialog": "~8.8.1", - "rc-drawer": "~4.4.2", - "rc-dropdown": "~3.5.0", - "rc-field-form": "~1.26.1", - "rc-image": "~5.6.0", - "rc-input": "~0.0.1-alpha.5", - "rc-input-number": "~7.3.0", - "rc-mentions": "~1.7.0", - "rc-menu": "~9.5.5", - "rc-motion": "^2.5.1", + "rc-collapse": "~3.3.0", + "rc-dialog": "~8.9.0", + "rc-drawer": "~5.1.0", + "rc-dropdown": "~4.0.0", + "rc-field-form": "~1.27.0", + "rc-image": "~5.7.0", + "rc-input": "~0.1.2", + "rc-input-number": "~7.3.9", + "rc-mentions": "~1.10.0", + "rc-menu": "~9.6.3", + "rc-motion": "^2.6.1", "rc-notification": "~4.6.0", - "rc-pagination": "~3.1.9", - "rc-picker": "~2.6.4", - "rc-progress": "~3.2.1", + "rc-pagination": "~3.1.17", + "rc-picker": "~2.6.11", + "rc-progress": "~3.3.2", "rc-rate": "~2.9.0", "rc-resize-observer": "^1.2.0", - "rc-segmented": "~2.1.0 ", - "rc-select": "~14.1.1", + "rc-segmented": "~2.1.0", + "rc-select": "~14.1.13", "rc-slider": "~10.0.0", "rc-steps": "~4.1.0", "rc-switch": "~3.2.0", - "rc-table": "~7.24.0", - "rc-tabs": "~11.13.0", - "rc-textarea": "~0.3.0", - "rc-tooltip": "~5.1.1", - "rc-tree": "~5.5.0", - "rc-tree-select": "~5.3.0", + "rc-table": "~7.26.0", + "rc-tabs": "~12.2.0", + "rc-textarea": "~0.4.5", + "rc-tooltip": "~5.2.0", + "rc-tree": "~5.7.0", + "rc-tree-select": "~5.5.0", "rc-trigger": "^5.2.10", "rc-upload": "~4.3.0", - "rc-util": "^5.20.0", + "rc-util": "^5.22.5", "scroll-into-view-if-needed": "^2.2.25" }, "funding": { @@ -4579,7 +4696,8 @@ }, "node_modules/array-tree-filter": { "version": "2.1.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz", + "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==" }, "node_modules/array-union": { "version": "2.1.0", @@ -4647,8 +4765,9 @@ "license": "MIT" }, "node_modules/async-validator": { - "version": "4.1.1", - "license": "MIT" + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz", + "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==" }, "node_modules/asynckit": { "version": "0.4.0", @@ -4674,7 +4793,9 @@ } }, "node_modules/autoprefixer": { - "version": "10.4.7", + "version": "10.4.12", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.12.tgz", + "integrity": "sha512-WrCGV9/b97Pa+jtwf5UGaRjgQIg7OK3D06GnoYoZNcG1Xb8Gt3EfuKjlhh9i/VtT16g6PYjZ69jdJ2g8FxSC4Q==", "dev": true, "funding": [ { @@ -4686,10 +4807,9 @@ "url": "https://tidelift.com/funding/github/npm/autoprefixer" } ], - "license": "MIT", "dependencies": { - "browserslist": "^4.20.3", - "caniuse-lite": "^1.0.30001335", + "browserslist": "^4.21.4", + "caniuse-lite": "^1.0.30001407", "fraction.js": "^4.2.0", "normalize-range": "^0.1.2", "picocolors": "^1.0.0", @@ -4836,6 +4956,15 @@ "object.assign": "^4.1.0" } }, + "node_modules/babel-plugin-import": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/babel-plugin-import/-/babel-plugin-import-1.13.5.tgz", + "integrity": "sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.0.0" + } + }, "node_modules/babel-plugin-istanbul": { "version": "6.1.1", "dev": true, @@ -5146,7 +5275,9 @@ "license": "BSD-2-Clause" }, "node_modules/browserslist": { - "version": "4.20.3", + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", "dev": true, "funding": [ { @@ -5158,13 +5289,11 @@ "url": "https://tidelift.com/funding/github/npm/browserslist" } ], - "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001332", - "electron-to-chromium": "^1.4.118", - "escalade": "^3.1.1", - "node-releases": "^2.0.3", - "picocolors": "^1.0.0" + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" }, "bin": { "browserslist": "cli.js" @@ -5270,7 +5399,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001336", + "version": "1.0.30001422", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001422.tgz", + "integrity": "sha512-hSesn02u1QacQHhaxl/kNMZwqVG35Sz/8DgvmgedxSH8z9UUpcDYSPYgsj3x5dQNRcNp6BwpSfQfVzYUTm+fog==", "dev": true, "funding": [ { @@ -5281,8 +5412,7 @@ "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" } - ], - "license": "CC-BY-4.0" + ] }, "node_modules/case-sensitive-paths-webpack-plugin": { "version": "2.4.0", @@ -5385,8 +5515,9 @@ "license": "MIT" }, "node_modules/classnames": { - "version": "2.3.1", - "license": "MIT" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" }, "node_modules/clean-css": { "version": "5.3.0", @@ -5484,6 +5615,20 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/clsx": { "version": "1.1.1", "license": "MIT", @@ -5591,9 +5736,10 @@ "license": "MIT" }, "node_modules/colord": { - "version": "2.9.2", - "dev": true, - "license": "MIT" + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", + "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", + "dev": true }, "node_modules/colorette": { "version": "2.0.16", @@ -5757,6 +5903,18 @@ "dev": true, "license": "MIT" }, + "node_modules/copy-anything": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "dependencies": { + "is-what": "^3.14.1" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/copy-to-clipboard": { "version": "3.3.1", "license": "MIT", @@ -5825,6 +5983,43 @@ "node": ">=10" } }, + "node_modules/cosmiconfig-typescript-loader": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.1.1.tgz", + "integrity": "sha512-9DHpa379Gp0o0Zefii35fcmuuin6q92FnLDffzdZ0l9tVd3nEobG3O+MZ06+kuBvFTSVScvNb/oHA13Nd4iipg==", + "dev": true, + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@types/node": "*", + "cosmiconfig": ">=7", + "ts-node": ">=10", + "typescript": ">=3" + } + }, + "node_modules/craco-less": { + "version": "2.1.0-alpha.0", + "resolved": "https://registry.npmjs.org/craco-less/-/craco-less-2.1.0-alpha.0.tgz", + "integrity": "sha512-1kj9Y7Y06Fbae3SJJtz1OvXsaKxjh0jTOwnvzKWOqrojQZbwC2K/d0dxDRUpHTDkIUmxbdzqMmI4LM9JfthQ6Q==", + "dev": true, + "dependencies": { + "less": "^4.1.1", + "less-loader": "^7.3.0" + }, + "peerDependencies": { + "@craco/craco": ">7.0.0-alpha", + "react-scripts": "^5.0.0" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "peer": true + }, "node_modules/cross-spawn": { "version": "7.0.3", "dev": true, @@ -6342,8 +6537,9 @@ } }, "node_modules/date-fns": { - "version": "2.28.0", - "license": "MIT", + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==", "engines": { "node": ">=0.11" }, @@ -6353,8 +6549,9 @@ } }, "node_modules/dayjs": { - "version": "1.11.1", - "license": "MIT" + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.5.tgz", + "integrity": "sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA==" }, "node_modules/debug": { "version": "4.3.4", @@ -6528,6 +6725,16 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "27.5.1", "dev": true, @@ -6731,9 +6938,10 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.134", - "dev": true, - "license": "ISC" + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", + "dev": true }, "node_modules/emittery": { "version": "0.8.1", @@ -6768,9 +6976,10 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.9.3", + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", "dev": true, - "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" @@ -6787,6 +6996,19 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, + "optional": true, + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, "node_modules/error-ex": { "version": "1.3.2", "dev": true, @@ -7059,6 +7281,62 @@ "ms": "^2.1.1" } }, + "node_modules/eslint-import-resolver-typescript": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.1.tgz", + "integrity": "sha512-U7LUjNJPYjNsHvAUAkt/RU3fcTSpbllA0//35B4eLYTX74frmOepbt7F7J3D1IGtj9k21buOpaqtDd4ZlS/BYQ==", + "dev": true, + "dependencies": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.10.0", + "get-tsconfig": "^4.2.0", + "globby": "^13.1.2", + "is-core-module": "^2.10.0", + "is-glob": "^4.0.3", + "synckit": "^0.8.3" + }, + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + }, + "peerDependencies": { + "eslint": "*", + "eslint-plugin-import": "*" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/globby": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", + "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "dev": true, + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint-import-resolver-typescript/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/eslint-module-utils": { "version": "2.7.3", "dev": true, @@ -7159,8 +7437,9 @@ }, "node_modules/eslint-plugin-import": { "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "dev": true, - "license": "MIT", "dependencies": { "array-includes": "^3.1.4", "array.prototype.flat": "^1.2.5", @@ -7230,6 +7509,19 @@ } } }, + "node_modules/eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "dev": true, + "dependencies": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + }, + "engines": { + "node": ">=12.0" + } + }, "node_modules/eslint-plugin-jsx-a11y": { "version": "6.5.1", "dev": true, @@ -7267,6 +7559,27 @@ "node": ">=6.0" } }, + "node_modules/eslint-plugin-prettier": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz", + "integrity": "sha512-f/0rXLXUt0oFYs8ra4w49wYZBG5GKZpAYsJSm6rnYL5uVDjd+zowwMwVZHnAjf4edNrKpCDYfXDgmRE/Ak7QyQ==", + "dev": true, + "dependencies": { + "prettier-linter-helpers": "^1.0.0" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "eslint": ">=7.28.0", + "prettier": ">=2.0.0" + }, + "peerDependenciesMeta": { + "eslint-config-prettier": { + "optional": true + } + } + }, "node_modules/eslint-plugin-react": { "version": "7.29.4", "dev": true, @@ -7716,10 +8029,17 @@ "version": "3.1.3", "license": "MIT" }, + "node_modules/fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, "node_modules/fast-glob": { - "version": "3.2.11", + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", "dev": true, - "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -8249,6 +8569,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-tsconfig": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.2.0.tgz", + "integrity": "sha512-X8u8fREiYOE6S8hLbq99PeykTDoLVnxvF4DjWKJmz9xy2nNRdUcV8ZN9tniJFeKyTU3qnC9lL8n4Chd6LmVKHg==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/glob": { "version": "7.2.0", "license": "ISC", @@ -8326,6 +8655,12 @@ "node": ">=4" } }, + "node_modules/globalyzer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/globalyzer/-/globalyzer-0.1.0.tgz", + "integrity": "sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q==", + "dev": true + }, "node_modules/globby": { "version": "11.1.0", "dev": true, @@ -8345,6 +8680,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/globrex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/globrex/-/globrex-0.1.2.tgz", + "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", + "dev": true + }, "node_modules/graceful-fs": { "version": "4.2.10", "dev": true, @@ -8485,11 +8826,6 @@ "wbuf": "^1.1.0" } }, - "node_modules/hpack.js/node_modules/isarray": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, "node_modules/hpack.js/node_modules/readable-stream": { "version": "2.3.7", "dev": true, @@ -8784,6 +9120,19 @@ "node": ">= 4" } }, + "node_modules/image-size": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true, + "optional": true, + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/immer": { "version": "9.0.12", "dev": true, @@ -8934,9 +9283,10 @@ } }, "node_modules/is-core-module": { - "version": "2.9.0", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "dev": true, - "license": "MIT", "dependencies": { "has": "^1.0.3" }, @@ -9064,6 +9414,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "dev": true, @@ -9166,6 +9528,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-what": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true + }, "node_modules/is-wsl": { "version": "2.2.0", "dev": true, @@ -9177,11 +9545,26 @@ "node": ">=8" } }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, "node_modules/isexe": { "version": "2.0.0", "dev": true, "license": "ISC" }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", "dev": true, @@ -10525,7 +10908,8 @@ }, "node_modules/json2mq": { "version": "0.2.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", + "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", "dependencies": { "string-convert": "^0.2.0" } @@ -10541,6 +10925,12 @@ "node": ">=6" } }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "node_modules/jsonfile": { "version": "6.1.0", "dev": true, @@ -10609,13 +10999,91 @@ "language-subtag-registry": "~0.3.2" } }, - "node_modules/leven": { - "version": "3.1.0", + "node_modules/less": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/less/-/less-4.1.3.tgz", + "integrity": "sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==", "dev": true, - "license": "MIT", + "dependencies": { + "copy-anything": "^2.0.1", + "parse-node-version": "^1.0.1", + "tslib": "^2.3.0" + }, + "bin": { + "lessc": "bin/lessc" + }, "engines": { "node": ">=6" - } + }, + "optionalDependencies": { + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "needle": "^3.1.0", + "source-map": "~0.6.0" + } + }, + "node_modules/less-loader": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-7.3.0.tgz", + "integrity": "sha512-Mi8915g7NMaLlgi77mgTTQvK022xKRQBIVDSyfl3ErTuBhmZBQab0mjeJjNNqGbdR+qrfTleKXqbGI4uEFavxg==", + "dev": true, + "dependencies": { + "klona": "^2.0.4", + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "less": "^3.5.0 || ^4.0.0", + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/less/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "optional": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/less/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "optional": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/less/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, + "node_modules/leven": { + "version": "3.1.0", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } }, "node_modules/levn": { "version": "0.4.1", @@ -11038,6 +11506,13 @@ "semver": "bin/semver.js" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "peer": true + }, "node_modules/makeerror": { "version": "1.0.12", "dev": true, @@ -11294,6 +11769,47 @@ "dev": true, "license": "MIT" }, + "node_modules/needle": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.1.0.tgz", + "integrity": "sha512-gCE9weDhjVGCRqS8dwDR/D3GTAeyXLXuqp7I8EzH6DllZGXSUyxuqqLh+YX9rMAWaaTFyVAg6rHGL25dqvczKw==", + "dev": true, + "optional": true, + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/needle/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/needle/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/negotiator": { "version": "0.6.3", "dev": true, @@ -11335,9 +11851,10 @@ "license": "MIT" }, "node_modules/node-releases": { - "version": "2.0.4", - "dev": true, - "license": "MIT" + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", + "dev": true }, "node_modules/normalize-path": { "version": "3.0.0", @@ -11696,6 +12213,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse-node-version": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.1.tgz", + "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/parse5": { "version": "6.0.1", "dev": true, @@ -11791,6 +12317,16 @@ "node": ">=0.10" } }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "optional": true, + "engines": { + "node": ">=6" + } + }, "node_modules/pirates": { "version": "4.0.5", "dev": true, @@ -11926,7 +12462,9 @@ } }, "node_modules/postcss": { - "version": "8.4.13", + "version": "8.4.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", + "integrity": "sha512-Wi8mWhncLJm11GATDaQKobXSNEYGUHeQLiQqDFG1qQ5UTDPTEvKw0Xt5NsTpktGTwLps3ByrWsBrG0rB8YQ9oA==", "dev": true, "funding": [ { @@ -11938,9 +12476,8 @@ "url": "https://tidelift.com/funding/github/npm/postcss" } ], - "license": "MIT", "dependencies": { - "nanoid": "^3.3.3", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" }, @@ -13083,6 +13620,18 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, + "node_modules/prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "dependencies": { + "fast-diff": "^1.1.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/pretty-bytes": { "version": "5.6.0", "dev": true, @@ -13186,6 +13735,13 @@ "node": ">= 0.10" } }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true, + "optional": true + }, "node_modules/psl": { "version": "1.8.0", "dev": true, @@ -13315,14 +13871,15 @@ } }, "node_modules/rc-cascader": { - "version": "3.5.0", - "license": "MIT", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz", + "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==", "dependencies": { "@babel/runtime": "^7.12.5", "array-tree-filter": "^2.1.0", "classnames": "^2.3.1", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.6.1" }, "peerDependencies": { @@ -13343,8 +13900,9 @@ } }, "node_modules/rc-collapse": { - "version": "3.1.4", - "license": "MIT", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz", + "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13358,8 +13916,9 @@ } }, "node_modules/rc-dialog": { - "version": "8.8.1", - "license": "MIT", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz", + "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -13372,11 +13931,14 @@ } }, "node_modules/rc-drawer": { - "version": "4.4.3", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz", + "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-util": "^5.7.0" + "rc-motion": "^2.6.1", + "rc-util": "^5.21.2" }, "peerDependencies": { "react": ">=16.9.0", @@ -13384,12 +13946,13 @@ } }, "node_modules/rc-dropdown": { - "version": "3.5.2", - "license": "MIT", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz", + "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==", "dependencies": { - "@babel/runtime": "^7.10.1", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", - "rc-trigger": "^5.0.4", + "rc-trigger": "^5.3.1", "rc-util": "^5.17.0" }, "peerDependencies": { @@ -13398,10 +13961,11 @@ } }, "node_modules/rc-field-form": { - "version": "1.26.3", - "license": "MIT", + "version": "1.27.3", + "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.3.tgz", + "integrity": "sha512-HGqxHnmGQgkPApEcikV4qTg3BLPC82uB/cwBDftDt1pYaqitJfSl5TFTTUMKVEJVT5RqJ2Zi68ME1HmIMX2HAw==", "dependencies": { - "@babel/runtime": "^7.8.4", + "@babel/runtime": "^7.18.0", "async-validator": "^4.1.0", "rc-util": "^5.8.0" }, @@ -13414,12 +13978,13 @@ } }, "node_modules/rc-image": { - "version": "5.6.2", - "license": "MIT", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz", + "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==", "dependencies": { "@babel/runtime": "^7.11.2", "classnames": "^2.2.6", - "rc-dialog": "~8.8.0", + "rc-dialog": "~8.9.0", "rc-util": "^5.0.6" }, "peerDependencies": { @@ -13428,8 +13993,9 @@ } }, "node_modules/rc-input": { - "version": "0.0.1-alpha.7", - "license": "MIT", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.4.tgz", + "integrity": "sha512-FqDdNz+fV2dKNgfXzcSLKvC+jEs1709t7nD+WdfjrdSaOcefpgc7BUJYadc3usaING+b7ediMTfKxuJBsEFbXA==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13441,12 +14007,13 @@ } }, "node_modules/rc-input-number": { - "version": "7.3.4", - "license": "MIT", + "version": "7.3.9", + "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz", + "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", - "rc-util": "^5.9.8" + "rc-util": "^5.23.0" }, "peerDependencies": { "react": ">=16.9.0", @@ -13454,15 +14021,16 @@ } }, "node_modules/rc-mentions": { - "version": "1.7.1", - "license": "MIT", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz", + "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-menu": "~9.5.1", - "rc-textarea": "^0.3.0", + "rc-menu": "~9.6.0", + "rc-textarea": "^0.4.0", "rc-trigger": "^5.0.4", - "rc-util": "^5.0.1" + "rc-util": "^5.22.5" }, "peerDependencies": { "react": ">=16.9.0", @@ -13470,8 +14038,9 @@ } }, "node_modules/rc-menu": { - "version": "9.5.5", - "license": "MIT", + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz", + "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13487,8 +14056,9 @@ } }, "node_modules/rc-motion": { - "version": "2.6.0", - "license": "MIT", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz", + "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13517,8 +14087,9 @@ } }, "node_modules/rc-overflow": { - "version": "1.2.5", - "license": "MIT", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz", + "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==", "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -13531,8 +14102,9 @@ } }, "node_modules/rc-pagination": { - "version": "3.1.16", - "license": "MIT", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz", + "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1" @@ -13543,8 +14115,9 @@ } }, "node_modules/rc-picker": { - "version": "2.6.8", - "license": "MIT", + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz", + "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -13564,8 +14137,9 @@ } }, "node_modules/rc-progress": { - "version": "3.2.4", - "license": "MIT", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz", + "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -13594,7 +14168,8 @@ }, "node_modules/rc-resize-observer": { "version": "1.2.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz", + "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -13621,8 +14196,9 @@ } }, "node_modules/rc-select": { - "version": "14.1.2", - "license": "MIT", + "version": "14.1.13", + "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz", + "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -13688,13 +14264,14 @@ } }, "node_modules/rc-table": { - "version": "7.24.1", - "license": "MIT", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz", + "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-resize-observer": "^1.1.0", - "rc-util": "^5.14.0", + "rc-util": "^5.22.5", "shallowequal": "^1.1.0" }, "engines": { @@ -13706,13 +14283,15 @@ } }, "node_modules/rc-tabs": { - "version": "11.13.0", - "license": "MIT", + "version": "12.2.1", + "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz", + "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==", "dependencies": { "@babel/runtime": "^7.11.2", "classnames": "2.x", - "rc-dropdown": "~3.5.0", - "rc-menu": "~9.5.1", + "rc-dropdown": "~4.0.0", + "rc-menu": "~9.6.0", + "rc-motion": "^2.6.2", "rc-resize-observer": "^1.0.0", "rc-util": "^5.5.0" }, @@ -13725,13 +14304,14 @@ } }, "node_modules/rc-textarea": { - "version": "0.3.7", - "license": "MIT", + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.6.tgz", + "integrity": "sha512-HEKCu8nouXXayqYelQnhQm8fdH7v92pAQvfVCz+jhIPv2PHTyBxVrmoZJMn3B8cU+wdyuvRGkshngO3/TzBn4w==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", "rc-resize-observer": "^1.0.0", - "rc-util": "^5.7.0", + "rc-util": "^5.24.4", "shallowequal": "^1.1.0" }, "peerDependencies": { @@ -13740,10 +14320,12 @@ } }, "node_modules/rc-tooltip": { - "version": "5.1.1", - "license": "MIT", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz", + "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==", "dependencies": { "@babel/runtime": "^7.11.2", + "classnames": "^2.3.1", "rc-trigger": "^5.0.0" }, "peerDependencies": { @@ -13752,14 +14334,15 @@ } }, "node_modules/rc-tree": { - "version": "5.5.0", - "license": "MIT", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz", + "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-util": "^5.16.1", - "rc-virtual-list": "^3.4.2" + "rc-virtual-list": "^3.4.8" }, "engines": { "node": ">=10.x" @@ -13770,13 +14353,14 @@ } }, "node_modules/rc-tree-select": { - "version": "5.3.0", - "license": "MIT", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.3.tgz", + "integrity": "sha512-gv8KyC6J7f9e50OkGk1ibF7v8vL+iaBnA8Ep/EVlMma2/tGdBQXO9xIvPjX8eQrZL5PjoeTUndNPM3cY3721ng==", "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.16.1" }, "peerDependencies": { @@ -13785,10 +14369,11 @@ } }, "node_modules/rc-trigger": { - "version": "5.2.18", - "license": "MIT", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.3.tgz", + "integrity": "sha512-IC4nuTSAME7RJSgwvHCNDQrIzhvGMKf6NDu5veX+zk1MG7i1UnwTWWthcP9WHw3+FZfP3oZGvkrHFPu/EGkFKw==", "dependencies": { - "@babel/runtime": "^7.11.2", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", "rc-align": "^4.0.0", "rc-motion": "^2.0.0", @@ -13816,10 +14401,11 @@ } }, "node_modules/rc-util": { - "version": "5.21.2", - "license": "MIT", + "version": "5.24.4", + "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz", + "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==", "dependencies": { - "@babel/runtime": "^7.12.5", + "@babel/runtime": "^7.18.3", "react-is": "^16.12.0", "shallowequal": "^1.1.0" }, @@ -13829,9 +14415,11 @@ } }, "node_modules/rc-virtual-list": { - "version": "3.4.7", - "license": "MIT", + "version": "3.4.11", + "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.11.tgz", + "integrity": "sha512-BvUUH60kkeTBPigN5F89HtGaA5jSP4y2aM6cJ4dk9Y42I9yY+h6i08wF6UKeDcxdfOU8j3I5HxkSS/xA77J3wA==", "dependencies": { + "@babel/runtime": "^7.20.0", "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.15.0" @@ -14052,6 +14640,18 @@ "node": ">=0.10.0" } }, + "node_modules/react-resizable": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/react-resizable/-/react-resizable-3.0.4.tgz", + "integrity": "sha512-StnwmiESiamNzdRHbSSvA65b0ZQJ7eVQpPusrSmcpyGKzC0gojhtO62xxH6YOBmepk9dQTBi9yxidL3W4s3EBA==", + "dependencies": { + "prop-types": "15.x", + "react-draggable": "^4.0.3" + }, + "peerDependencies": { + "react": ">= 16.3" + } + }, "node_modules/react-router": { "version": "6.3.0", "license": "MIT", @@ -14228,8 +14828,9 @@ } }, "node_modules/regenerator-runtime": { - "version": "0.13.9", - "license": "MIT" + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" }, "node_modules/regenerator-transform": { "version": "0.15.0", @@ -14962,6 +15563,18 @@ "dev": true, "license": "ISC" }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/shallowequal": { "version": "1.1.0", "license": "MIT" @@ -15231,7 +15844,8 @@ }, "node_modules/string-convert": { "version": "0.2.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", + "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==" }, "node_modules/string-length": { "version": "4.0.2", @@ -15424,9 +16038,10 @@ } }, "node_modules/supports-hyperlinks": { - "version": "2.2.0", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", "dev": true, - "license": "MIT", "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -15538,6 +16153,28 @@ "dev": true, "license": "MIT" }, + "node_modules/synckit": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.4.tgz", + "integrity": "sha512-Dn2ZkzMdSX827QbowGbU/4yjWuvNaCoScLLoMo/yKbu+P4GBR6cRGKZH27k6a9bRzdqcyd1DE96pQtQ6uNkmyw==", + "dev": true, + "dependencies": { + "@pkgr/utils": "^2.3.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/synckit/node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + }, "node_modules/tailwindcss": { "version": "3.0.24", "dev": true, @@ -15724,6 +16361,16 @@ "dev": true, "license": "MIT" }, + "node_modules/tiny-glob": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", + "integrity": "sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==", + "dev": true, + "dependencies": { + "globalyzer": "0.1.0", + "globrex": "^0.1.2" + } + }, "node_modules/tmpl": { "version": "1.0.5", "dev": true, @@ -15794,6 +16441,67 @@ "dev": true, "license": "MIT" }, + "node_modules/ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "peer": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/ts-node/node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "peer": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ts-node/node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "peer": true + }, "node_modules/tsconfig-paths": { "version": "3.14.1", "dev": true, @@ -15996,6 +16704,32 @@ "yarn": "*" } }, + "node_modules/update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "dev": true, @@ -16049,6 +16783,13 @@ "dev": true, "license": "MIT" }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "peer": true + }, "node_modules/v8-to-istanbul": { "version": "8.1.1", "dev": true, @@ -16078,6 +16819,43 @@ "node": ">= 0.8" } }, + "node_modules/vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "dev": true, + "dependencies": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.7.tgz", + "integrity": "sha512-bFJH7UQxlXT8kKeyiyu41r22jCZXG8kuuVVA33OEJn1diWOZK5n8zBSPZFHVBOu8kXZ6h0LIRhf5UnCo61J4Hg==", + "dev": true + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.2", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.2.tgz", + "integrity": "sha512-zHhCWatviizPIq9B7Vh9uvrH6x3sK8itC84HkamnBWoDFJtzBf7SWlpLCZUit72b3os45h6RWQNC9xHRDF8dRA==", + "dev": true + }, + "node_modules/vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "dev": true + }, + "node_modules/vscode-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.6.tgz", + "integrity": "sha512-fmL7V1eiDBFRRnu+gfRWTzyPpNIHJTc4mWnFkwBUmO9U3KPgJAmTx7oxi2bl/Rh6HLdU7+4C9wlj0k2E4AdKFQ==", + "dev": true + }, "node_modules/w3c-hr-time": { "version": "1.0.2", "dev": true, @@ -16137,8 +16915,9 @@ }, "node_modules/webpack": { "version": "5.72.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.72.0.tgz", + "integrity": "sha512-qmSmbspI0Qo5ld49htys8GY9XhS9CGqFoHTsOVAnjBdg0Zn79y135R+k4IR4rKK6+eKaabMhJwiVB7xw0SJu5w==", "dev": true, - "license": "MIT", "dependencies": { "@types/eslint-scope": "^3.7.3", "@types/estree": "^0.0.51", @@ -16357,6 +17136,19 @@ "node": ">=10.13.0" } }, + "node_modules/webpack-merge": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", + "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", + "dev": true, + "dependencies": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/webpack-sources": { "version": "3.2.3", "dev": true, @@ -16463,6 +17255,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, "node_modules/word-wrap": { "version": "1.2.3", "dev": true, @@ -16813,6 +17611,16 @@ "node": ">=10" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "dev": true, @@ -16842,6 +17650,8 @@ }, "@ant-design/icons": { "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@ant-design/icons/-/icons-4.7.0.tgz", + "integrity": "sha512-aoB4Z7JA431rt6d4u+8xcNPPCrdufSRMUOpxa1ab6mz1JCQZOEVolj2WVs/tDFmN62zzK30mNelEsprLYsSF3g==", "requires": { "@ant-design/colors": "^6.0.0", "@ant-design/icons-svg": "^4.2.1", @@ -16854,13 +17664,15 @@ "version": "4.2.1" }, "@ant-design/react-slick": { - "version": "0.28.4", + "version": "0.29.2", + "resolved": "https://registry.npmjs.org/@ant-design/react-slick/-/react-slick-0.29.2.tgz", + "integrity": "sha512-kgjtKmkGHa19FW21lHnAfyyH9AAoh35pBdcJ53rHmQ3O+cfFHGHnUbj/HFrRNJ5vIts09FKJVAD8RpaC+RaWfA==", "requires": { "@babel/runtime": "^7.10.4", "classnames": "^2.2.5", "json2mq": "^0.2.0", "lodash": "^4.17.21", - "resize-observer-polyfill": "^1.5.0" + "resize-observer-polyfill": "^1.5.1" } }, "@apideck/better-ajv-errors": { @@ -17964,9 +18776,11 @@ } }, "@babel/runtime": { - "version": "7.17.9", + "version": "7.20.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.20.0.tgz", + "integrity": "sha512-NDYdls71fTXoU8TZHfbBWg7DiZfNzClcKui/+kyi6ppD2L1qnWW3VV6CjtaBXSUGGhiTWJ6ereOIkUvenif66Q==", "requires": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.13.10" } }, "@babel/runtime-corejs3": { @@ -18014,6 +18828,31 @@ "version": "0.2.3", "dev": true }, + "@craco/craco": { + "version": "7.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@craco/craco/-/craco-7.0.0-alpha.8.tgz", + "integrity": "sha512-IN3/ldPaktGflPu342cg7n8LYa2c3x9H2XzngUkDzTjro25ig1GyVcUdnG1U0X6wrRTF9K1AxZ5su9jLbdyFUw==", + "dev": true, + "requires": { + "autoprefixer": "^10.4.12", + "cosmiconfig": "^7.0.1", + "cosmiconfig-typescript-loader": "^4.1.1", + "cross-spawn": "^7.0.3", + "lodash": "^4.17.21", + "semver": "^7.3.7", + "webpack-merge": "^5.8.0" + } + }, + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "peer": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@csstools/normalize.css": { "version": "12.0.0", "dev": true @@ -18513,6 +19352,28 @@ "fastq": "^1.6.0" } }, + "@pkgr/utils": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.3.1.tgz", + "integrity": "sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.3", + "is-glob": "^4.0.3", + "open": "^8.4.0", + "picocolors": "^1.0.0", + "tiny-glob": "^0.2.9", + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, "@pmmmwh/react-refresh-webpack-plugin": { "version": "0.5.5", "dev": true, @@ -18767,6 +19628,34 @@ "version": "0.2.0", "dev": true }, + "@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", + "dev": true, + "peer": true + }, + "@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "peer": true + }, + "@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "peer": true + }, + "@tsconfig/node16": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.3.tgz", + "integrity": "sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==", + "dev": true, + "peer": true + }, "@types/aria-query": { "version": "4.2.2", "dev": true @@ -18991,6 +19880,15 @@ "redux": "^4.0.0" } }, + "@types/react-resizable": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/react-resizable/-/react-resizable-3.0.3.tgz", + "integrity": "sha512-W/QsUOZoXBAIBQNhNm95A5ohoaiUA874lWQytO2UP9dOjp5JHO9+a0cwYNabea7sA12ZDJnGVUFZxcNaNksAWA==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, "@types/resolve": { "version": "1.17.1", "dev": true, @@ -19571,51 +20469,53 @@ } }, "antd": { - "version": "4.20.2", + "version": "4.23.6", + "resolved": "https://registry.npmjs.org/antd/-/antd-4.23.6.tgz", + "integrity": "sha512-AYH57cWBDe1ChtbnvG8i9dpKG4WnjE3AG0zIKpXByFNnxsr4saV6/19ihE8/ImSGpohN4E2zTXmo7R5/MyVRKQ==", "requires": { "@ant-design/colors": "^6.0.0", "@ant-design/icons": "^4.7.0", - "@ant-design/react-slick": "~0.28.1", - "@babel/runtime": "^7.12.5", + "@ant-design/react-slick": "~0.29.1", + "@babel/runtime": "^7.18.3", "@ctrl/tinycolor": "^3.4.0", "classnames": "^2.2.6", "copy-to-clipboard": "^3.2.0", "lodash": "^4.17.21", "memoize-one": "^6.0.0", "moment": "^2.29.2", - "rc-cascader": "~3.5.0", + "rc-cascader": "~3.7.0", "rc-checkbox": "~2.3.0", - "rc-collapse": "~3.1.0", - "rc-dialog": "~8.8.1", - "rc-drawer": "~4.4.2", - "rc-dropdown": "~3.5.0", - "rc-field-form": "~1.26.1", - "rc-image": "~5.6.0", - "rc-input": "~0.0.1-alpha.5", - "rc-input-number": "~7.3.0", - "rc-mentions": "~1.7.0", - "rc-menu": "~9.5.5", - "rc-motion": "^2.5.1", + "rc-collapse": "~3.3.0", + "rc-dialog": "~8.9.0", + "rc-drawer": "~5.1.0", + "rc-dropdown": "~4.0.0", + "rc-field-form": "~1.27.0", + "rc-image": "~5.7.0", + "rc-input": "~0.1.2", + "rc-input-number": "~7.3.9", + "rc-mentions": "~1.10.0", + "rc-menu": "~9.6.3", + "rc-motion": "^2.6.1", "rc-notification": "~4.6.0", - "rc-pagination": "~3.1.9", - "rc-picker": "~2.6.4", - "rc-progress": "~3.2.1", + "rc-pagination": "~3.1.17", + "rc-picker": "~2.6.11", + "rc-progress": "~3.3.2", "rc-rate": "~2.9.0", "rc-resize-observer": "^1.2.0", - "rc-segmented": "~2.1.0 ", - "rc-select": "~14.1.1", + "rc-segmented": "~2.1.0", + "rc-select": "~14.1.13", "rc-slider": "~10.0.0", "rc-steps": "~4.1.0", "rc-switch": "~3.2.0", - "rc-table": "~7.24.0", - "rc-tabs": "~11.13.0", - "rc-textarea": "~0.3.0", - "rc-tooltip": "~5.1.1", - "rc-tree": "~5.5.0", - "rc-tree-select": "~5.3.0", + "rc-table": "~7.26.0", + "rc-tabs": "~12.2.0", + "rc-textarea": "~0.4.5", + "rc-tooltip": "~5.2.0", + "rc-tree": "~5.7.0", + "rc-tree-select": "~5.5.0", "rc-trigger": "^5.2.10", "rc-upload": "~4.3.0", - "rc-util": "^5.20.0", + "rc-util": "^5.22.5", "scroll-into-view-if-needed": "^2.2.25" } }, @@ -19658,7 +20558,9 @@ } }, "array-tree-filter": { - "version": "2.1.0" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-tree-filter/-/array-tree-filter-2.1.0.tgz", + "integrity": "sha512-4ROwICNlNw/Hqa9v+rk5h22KjmzB1JGTMVKP2AKJBOCgb0yL0ASf0+YvCcLNNwquOHNX48jkeZIJ3a+oOQqKcw==" }, "array-union": { "version": "2.1.0", @@ -19701,7 +20603,9 @@ "dev": true }, "async-validator": { - "version": "4.1.1" + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/async-validator/-/async-validator-4.2.5.tgz", + "integrity": "sha512-7HhHjtERjqlNbZtqNqy2rckN/SpOOlmDliet+lP7k+eKZEjPk3DgyeU9lIXLdeLz0uBbbVp+9Qdow9wJWgwwfg==" }, "asynckit": { "version": "0.4.0" @@ -19715,11 +20619,13 @@ "dev": true }, "autoprefixer": { - "version": "10.4.7", + "version": "10.4.12", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.12.tgz", + "integrity": "sha512-WrCGV9/b97Pa+jtwf5UGaRjgQIg7OK3D06GnoYoZNcG1Xb8Gt3EfuKjlhh9i/VtT16g6PYjZ69jdJ2g8FxSC4Q==", "dev": true, "requires": { - "browserslist": "^4.20.3", - "caniuse-lite": "^1.0.30001335", + "browserslist": "^4.21.4", + "caniuse-lite": "^1.0.30001407", "fraction.js": "^4.2.0", "normalize-range": "^0.1.2", "picocolors": "^1.0.0", @@ -19818,6 +20724,15 @@ "object.assign": "^4.1.0" } }, + "babel-plugin-import": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/babel-plugin-import/-/babel-plugin-import-1.13.5.tgz", + "integrity": "sha512-IkqnoV+ov1hdJVofly9pXRJmeDm9EtROfrc5i6eII0Hix2xMs5FEm8FG3ExMvazbnZBbgHIt6qdO8And6lCloQ==", + "dev": true, + "requires": { + "@babel/helper-module-imports": "^7.0.0" + } + }, "babel-plugin-istanbul": { "version": "6.1.1", "dev": true, @@ -20048,14 +20963,15 @@ "dev": true }, "browserslist": { - "version": "4.20.3", + "version": "4.21.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", + "integrity": "sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001332", - "electron-to-chromium": "^1.4.118", - "escalade": "^3.1.1", - "node-releases": "^2.0.3", - "picocolors": "^1.0.0" + "caniuse-lite": "^1.0.30001400", + "electron-to-chromium": "^1.4.251", + "node-releases": "^2.0.6", + "update-browserslist-db": "^1.0.9" } }, "bser": { @@ -20122,7 +21038,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001336", + "version": "1.0.30001422", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001422.tgz", + "integrity": "sha512-hSesn02u1QacQHhaxl/kNMZwqVG35Sz/8DgvmgedxSH8z9UUpcDYSPYgsj3x5dQNRcNp6BwpSfQfVzYUTm+fog==", "dev": true }, "case-sensitive-paths-webpack-plugin": { @@ -20188,7 +21106,9 @@ "version": "5.0.3" }, "classnames": { - "version": "2.3.1" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", + "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" }, "clean-css": { "version": "5.3.0", @@ -20247,6 +21167,17 @@ "wrap-ansi": "^7.0.0" } }, + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, "clsx": { "version": "1.1.1" }, @@ -20319,7 +21250,9 @@ "dev": true }, "colord": { - "version": "2.9.2", + "version": "2.9.3", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz", + "integrity": "sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==", "dev": true }, "colorette": { @@ -20427,6 +21360,15 @@ "version": "1.0.6", "dev": true }, + "copy-anything": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "requires": { + "is-what": "^3.14.1" + } + }, "copy-to-clipboard": { "version": "3.3.1", "requires": { @@ -20470,6 +21412,30 @@ "yaml": "^1.10.0" } }, + "cosmiconfig-typescript-loader": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/cosmiconfig-typescript-loader/-/cosmiconfig-typescript-loader-4.1.1.tgz", + "integrity": "sha512-9DHpa379Gp0o0Zefii35fcmuuin6q92FnLDffzdZ0l9tVd3nEobG3O+MZ06+kuBvFTSVScvNb/oHA13Nd4iipg==", + "dev": true, + "requires": {} + }, + "craco-less": { + "version": "2.1.0-alpha.0", + "resolved": "https://registry.npmjs.org/craco-less/-/craco-less-2.1.0-alpha.0.tgz", + "integrity": "sha512-1kj9Y7Y06Fbae3SJJtz1OvXsaKxjh0jTOwnvzKWOqrojQZbwC2K/d0dxDRUpHTDkIUmxbdzqMmI4LM9JfthQ6Q==", + "dev": true, + "requires": { + "less": "^4.1.1", + "less-loader": "^7.3.0" + } + }, + "create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "peer": true + }, "cross-spawn": { "version": "7.0.3", "dev": true, @@ -20780,10 +21746,14 @@ } }, "date-fns": { - "version": "2.28.0" + "version": "2.29.3", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.29.3.tgz", + "integrity": "sha512-dDCnyH2WnnKusqvZZ6+jA1O51Ibt8ZMRNkDZdyAyK4YfbDwa/cEmuztzG5pk6hqlp9aSBPYcjOlktquahGwGeA==" }, "dayjs": { - "version": "1.11.1" + "version": "1.11.5", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.5.tgz", + "integrity": "sha512-CAdX5Q3YW3Gclyo5Vpqkgpj8fSdLQcRuzfX6mC6Phy0nfJ0eGYOeS7m4mt2plDWLAtA4TqTakvbboHvUxfe4iA==" }, "debug": { "version": "4.3.4", @@ -20887,6 +21857,13 @@ "version": "1.2.2", "dev": true }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "peer": true + }, "diff-sequences": { "version": "27.5.1", "dev": true @@ -21028,7 +22005,9 @@ } }, "electron-to-chromium": { - "version": "1.4.134", + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, "emittery": { @@ -21048,7 +22027,9 @@ "dev": true }, "enhanced-resolve": { - "version": "5.9.3", + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.10.0.tgz", + "integrity": "sha512-T0yTFjdpldGY8PmuXXR0PyQ1ufZpEGiHVrp7zHKB7jdR4qlmZHhONVM5AQOAWXuF/w3dnHbEQVrNptJgt7F+cQ==", "dev": true, "requires": { "graceful-fs": "^4.2.4", @@ -21059,6 +22040,16 @@ "version": "2.2.0", "dev": true }, + "errno": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, + "optional": true, + "requires": { + "prr": "~1.0.1" + } + }, "error-ex": { "version": "1.3.2", "dev": true, @@ -21308,6 +22299,42 @@ } } }, + "eslint-import-resolver-typescript": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-typescript/-/eslint-import-resolver-typescript-3.5.1.tgz", + "integrity": "sha512-U7LUjNJPYjNsHvAUAkt/RU3fcTSpbllA0//35B4eLYTX74frmOepbt7F7J3D1IGtj9k21buOpaqtDd4ZlS/BYQ==", + "dev": true, + "requires": { + "debug": "^4.3.4", + "enhanced-resolve": "^5.10.0", + "get-tsconfig": "^4.2.0", + "globby": "^13.1.2", + "is-core-module": "^2.10.0", + "is-glob": "^4.0.3", + "synckit": "^0.8.3" + }, + "dependencies": { + "globby": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", + "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "dev": true, + "requires": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + } + }, + "slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true + } + } + }, "eslint-module-utils": { "version": "2.7.3", "dev": true, @@ -21372,6 +22399,8 @@ }, "eslint-plugin-import": { "version": "2.26.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.26.0.tgz", + "integrity": "sha512-hYfi3FXaM8WPLf4S1cikh/r4IxnO6zrhZbEGz2b660EJRbuxgpDS5gkCuYgGWg2xxh2rBuIr4Pvhve/7c31koA==", "dev": true, "requires": { "array-includes": "^3.1.4", @@ -21416,6 +22445,16 @@ "@typescript-eslint/experimental-utils": "^5.0.0" } }, + "eslint-plugin-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-3.1.0.tgz", + "integrity": "sha512-MrlG2ynFEHe7wDGwbUuFPsaT2b1uhuEFhJ+W1f1u+1C2EkXmTYJp4B1aAdQQ8M+CC3t//N/oRKiIVw14L2HR1g==", + "dev": true, + "requires": { + "lodash": "^4.17.21", + "vscode-json-languageservice": "^4.1.6" + } + }, "eslint-plugin-jsx-a11y": { "version": "6.5.1", "dev": true, @@ -21444,6 +22483,15 @@ } } }, + "eslint-plugin-prettier": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.2.1.tgz", + "integrity": "sha512-f/0rXLXUt0oFYs8ra4w49wYZBG5GKZpAYsJSm6rnYL5uVDjd+zowwMwVZHnAjf4edNrKpCDYfXDgmRE/Ak7QyQ==", + "dev": true, + "requires": { + "prettier-linter-helpers": "^1.0.0" + } + }, "eslint-plugin-react": { "version": "7.29.4", "dev": true, @@ -21678,8 +22726,16 @@ "fast-deep-equal": { "version": "3.1.3" }, + "fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, "fast-glob": { - "version": "3.2.11", + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", "dev": true, "requires": { "@nodelib/fs.stat": "^2.0.2", @@ -22012,6 +23068,12 @@ "get-intrinsic": "^1.1.1" } }, + "get-tsconfig": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.2.0.tgz", + "integrity": "sha512-X8u8fREiYOE6S8hLbq99PeykTDoLVnxvF4DjWKJmz9xy2nNRdUcV8ZN9tniJFeKyTU3qnC9lL8n4Chd6LmVKHg==", + "dev": true + }, "glob": { "version": "7.2.0", "requires": { @@ -22063,6 +23125,12 @@ "version": "11.12.0", "dev": true }, + "globalyzer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/globalyzer/-/globalyzer-0.1.0.tgz", + "integrity": "sha512-40oNTM9UfG6aBmuKxk/giHn5nQ8RVz/SS4Ir6zgzOv9/qC3kKZ9v4etGTcJbEl/NyVQH7FGU7d+X1egr57Md2Q==", + "dev": true + }, "globby": { "version": "11.1.0", "dev": true, @@ -22075,6 +23143,12 @@ "slash": "^3.0.0" } }, + "globrex": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/globrex/-/globrex-0.1.2.tgz", + "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", + "dev": true + }, "graceful-fs": { "version": "4.2.10", "dev": true @@ -22163,10 +23237,6 @@ "wbuf": "^1.1.0" }, "dependencies": { - "isarray": { - "version": "1.0.0", - "dev": true - }, "readable-stream": { "version": "2.3.7", "dev": true, @@ -22352,6 +23422,13 @@ "version": "5.2.0", "dev": true }, + "image-size": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true, + "optional": true + }, "immer": { "version": "9.0.12", "dev": true @@ -22438,7 +23515,9 @@ "dev": true }, "is-core-module": { - "version": "2.9.0", + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.11.0.tgz", + "integrity": "sha512-RRjxlvLDkD1YJwDbroBHMb+cukurkDWNyHx7D3oNB5x9rb5ogcksMC5wHCadcXoo67gVr/+3GFySh3134zi6rw==", "dev": true, "requires": { "has": "^1.0.3" @@ -22501,6 +23580,15 @@ "version": "3.0.0", "dev": true }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "requires": { + "isobject": "^3.0.1" + } + }, "is-potential-custom-element-name": { "version": "1.0.1", "dev": true @@ -22557,6 +23645,12 @@ "call-bind": "^1.0.2" } }, + "is-what": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true + }, "is-wsl": { "version": "2.2.0", "dev": true, @@ -22564,10 +23658,22 @@ "is-docker": "^2.0.0" } }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, "isexe": { "version": "2.0.0", "dev": true }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true + }, "istanbul-lib-coverage": { "version": "3.2.0", "dev": true @@ -23499,6 +24605,8 @@ }, "json2mq": { "version": "0.2.0", + "resolved": "https://registry.npmjs.org/json2mq/-/json2mq-0.2.0.tgz", + "integrity": "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==", "requires": { "string-convert": "^0.2.0" } @@ -23507,6 +24615,12 @@ "version": "2.2.1", "dev": true }, + "jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, "jsonfile": { "version": "6.1.0", "dev": true, @@ -23550,6 +24664,61 @@ "language-subtag-registry": "~0.3.2" } }, + "less": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/less/-/less-4.1.3.tgz", + "integrity": "sha512-w16Xk/Ta9Hhyei0Gpz9m7VS8F28nieJaL/VyShID7cYvP6IL5oHeL6p4TXSDJqZE/lNv0oJ2pGVjJsRkfwm5FA==", + "dev": true, + "requires": { + "copy-anything": "^2.0.1", + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "needle": "^3.1.0", + "parse-node-version": "^1.0.1", + "source-map": "~0.6.0", + "tslib": "^2.3.0" + }, + "dependencies": { + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "optional": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "optional": true + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, + "less-loader": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/less-loader/-/less-loader-7.3.0.tgz", + "integrity": "sha512-Mi8915g7NMaLlgi77mgTTQvK022xKRQBIVDSyfl3ErTuBhmZBQab0mjeJjNNqGbdR+qrfTleKXqbGI4uEFavxg==", + "dev": true, + "requires": { + "klona": "^2.0.4", + "loader-utils": "^2.0.0", + "schema-utils": "^3.0.0" + } + }, "leven": { "version": "3.1.0", "dev": true @@ -23812,6 +24981,13 @@ } } }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "peer": true + }, "makeerror": { "version": "1.0.12", "dev": true, @@ -23970,6 +25146,40 @@ "version": "1.4.0", "dev": true }, + "needle": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/needle/-/needle-3.1.0.tgz", + "integrity": "sha512-gCE9weDhjVGCRqS8dwDR/D3GTAeyXLXuqp7I8EzH6DllZGXSUyxuqqLh+YX9rMAWaaTFyVAg6rHGL25dqvczKw==", + "dev": true, + "optional": true, + "requires": { + "debug": "^3.2.6", + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "dev": true, + "optional": true, + "requires": { + "ms": "^2.1.1" + } + }, + "iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + } + } + } + }, "negotiator": { "version": "0.6.3", "dev": true @@ -24001,7 +25211,9 @@ "dev": true }, "node-releases": { - "version": "2.0.4", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", "dev": true }, "normalize-path": { @@ -24219,6 +25431,12 @@ "lines-and-columns": "^1.1.6" } }, + "parse-node-version": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parse-node-version/-/parse-node-version-1.0.1.tgz", + "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", + "dev": true + }, "parse5": { "version": "6.0.1", "dev": true @@ -24276,6 +25494,13 @@ "version": "0.6.0", "dev": true }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "optional": true + }, "pirates": { "version": "4.0.5", "dev": true @@ -24361,10 +25586,12 @@ } }, "postcss": { - "version": "8.4.13", + "version": "8.4.18", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.18.tgz", + "integrity": "sha512-Wi8mWhncLJm11GATDaQKobXSNEYGUHeQLiQqDFG1qQ5UTDPTEvKw0Xt5NsTpktGTwLps3ByrWsBrG0rB8YQ9oA==", "dev": true, "requires": { - "nanoid": "^3.3.3", + "nanoid": "^3.3.4", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } @@ -24962,6 +26189,15 @@ "version": "2.7.1", "dev": true }, + "prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "requires": { + "fast-diff": "^1.1.2" + } + }, "pretty-bytes": { "version": "5.6.0", "dev": true @@ -25034,6 +26270,13 @@ } } }, + "prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true, + "optional": true + }, "psl": { "version": "1.8.0", "dev": true @@ -25107,13 +26350,15 @@ } }, "rc-cascader": { - "version": "3.5.0", + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/rc-cascader/-/rc-cascader-3.7.0.tgz", + "integrity": "sha512-SFtGpwmYN7RaWEAGTS4Rkc62ZV/qmQGg/tajr/7mfIkleuu8ro9Hlk6J+aA0x1YS4zlaZBtTcSaXM01QMiEV/A==", "requires": { "@babel/runtime": "^7.12.5", "array-tree-filter": "^2.1.0", "classnames": "^2.3.1", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.6.1" } }, @@ -25125,7 +26370,9 @@ } }, "rc-collapse": { - "version": "3.1.4", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/rc-collapse/-/rc-collapse-3.3.1.tgz", + "integrity": "sha512-cOJfcSe3R8vocrF8T+PgaHDrgeA1tX+lwfhwSj60NX9QVRidsILIbRNDLD6nAzmcvVC5PWiIRiR4S1OobxdhCg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25135,7 +26382,9 @@ } }, "rc-dialog": { - "version": "8.8.1", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/rc-dialog/-/rc-dialog-8.9.0.tgz", + "integrity": "sha512-Cp0tbJnrvPchJfnwIvOMWmJ4yjX3HWFatO6oBFD1jx8QkgsQCR0p8nUWAKdd3seLJhEC39/v56kZaEjwp9muoQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -25144,41 +26393,52 @@ } }, "rc-drawer": { - "version": "4.4.3", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/rc-drawer/-/rc-drawer-5.1.0.tgz", + "integrity": "sha512-pU3Tsn99pxGdYowXehzZbdDVE+4lDXSGb7p8vA9mSmr569oc2Izh4Zw5vLKSe/Xxn2p5MSNbLVqD4tz+pK6SOw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-util": "^5.7.0" + "rc-motion": "^2.6.1", + "rc-util": "^5.21.2" } }, "rc-dropdown": { - "version": "3.5.2", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/rc-dropdown/-/rc-dropdown-4.0.1.tgz", + "integrity": "sha512-OdpXuOcme1rm45cR0Jzgfl1otzmU4vuBVb+etXM8vcaULGokAKVpKlw8p6xzspG7jGd/XxShvq+N3VNEfk/l5g==", "requires": { - "@babel/runtime": "^7.10.1", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", - "rc-trigger": "^5.0.4", + "rc-trigger": "^5.3.1", "rc-util": "^5.17.0" } }, "rc-field-form": { - "version": "1.26.3", + "version": "1.27.3", + "resolved": "https://registry.npmjs.org/rc-field-form/-/rc-field-form-1.27.3.tgz", + "integrity": "sha512-HGqxHnmGQgkPApEcikV4qTg3BLPC82uB/cwBDftDt1pYaqitJfSl5TFTTUMKVEJVT5RqJ2Zi68ME1HmIMX2HAw==", "requires": { - "@babel/runtime": "^7.8.4", + "@babel/runtime": "^7.18.0", "async-validator": "^4.1.0", "rc-util": "^5.8.0" } }, "rc-image": { - "version": "5.6.2", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/rc-image/-/rc-image-5.7.1.tgz", + "integrity": "sha512-QyMfdhoUfb5W14plqXSisaYwpdstcLYnB0MjX5ccIK2rydQM9sDPuekQWu500DDGR2dBaIF5vx9XbWkNFK17Fg==", "requires": { "@babel/runtime": "^7.11.2", "classnames": "^2.2.6", - "rc-dialog": "~8.8.0", + "rc-dialog": "~8.9.0", "rc-util": "^5.0.6" } }, "rc-input": { - "version": "0.0.1-alpha.7", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/rc-input/-/rc-input-0.1.4.tgz", + "integrity": "sha512-FqDdNz+fV2dKNgfXzcSLKvC+jEs1709t7nD+WdfjrdSaOcefpgc7BUJYadc3usaING+b7ediMTfKxuJBsEFbXA==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25186,26 +26446,32 @@ } }, "rc-input-number": { - "version": "7.3.4", + "version": "7.3.9", + "resolved": "https://registry.npmjs.org/rc-input-number/-/rc-input-number-7.3.9.tgz", + "integrity": "sha512-u0+miS+SATdb6DtssYei2JJ1WuZME+nXaG6XGtR8maNyW5uGDytfDu60OTWLQEb0Anv/AcCzehldV8CKmKyQfA==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", - "rc-util": "^5.9.8" + "rc-util": "^5.23.0" } }, "rc-mentions": { - "version": "1.7.1", + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/rc-mentions/-/rc-mentions-1.10.0.tgz", + "integrity": "sha512-oMlYWnwXSxP2NQVlgxOTzuG/u9BUc3ySY78K3/t7MNhJWpZzXTao+/Bic6tyZLuNCO89//hVQJBdaR2rnFQl6Q==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", - "rc-menu": "~9.5.1", - "rc-textarea": "^0.3.0", + "rc-menu": "~9.6.0", + "rc-textarea": "^0.4.0", "rc-trigger": "^5.0.4", - "rc-util": "^5.0.1" + "rc-util": "^5.22.5" } }, "rc-menu": { - "version": "9.5.5", + "version": "9.6.4", + "resolved": "https://registry.npmjs.org/rc-menu/-/rc-menu-9.6.4.tgz", + "integrity": "sha512-6DiNAjxjVIPLZXHffXxxcyE15d4isRL7iQ1ru4MqYDH2Cqc5bW96wZOdMydFtGLyDdnmEQ9jVvdCE9yliGvzkw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25217,7 +26483,9 @@ } }, "rc-motion": { - "version": "2.6.0", + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/rc-motion/-/rc-motion-2.6.2.tgz", + "integrity": "sha512-4w1FaX3dtV749P8GwfS4fYnFG4Rb9pxvCYPc/b2fw1cmlHJWNNgOFIz7ysiD+eOrzJSvnLJWlNQQncpNMXwwpg==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25234,7 +26502,9 @@ } }, "rc-overflow": { - "version": "1.2.5", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc-overflow/-/rc-overflow-1.2.8.tgz", + "integrity": "sha512-QJ0UItckWPQ37ZL1dMEBAdY1dhfTXFL9k6oTTcyydVwoUNMnMqCGqnRNA98axSr/OeDKqR6DVFyi8eA5RQI/uQ==", "requires": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", @@ -25243,14 +26513,18 @@ } }, "rc-pagination": { - "version": "3.1.16", + "version": "3.1.17", + "resolved": "https://registry.npmjs.org/rc-pagination/-/rc-pagination-3.1.17.tgz", + "integrity": "sha512-/BQ5UxcBnW28vFAcP2hfh+Xg15W0QZn8TWYwdCApchMH1H0CxiaUUcULP8uXcFM1TygcdKWdt3JqsL9cTAfdkQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1" } }, "rc-picker": { - "version": "2.6.8", + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/rc-picker/-/rc-picker-2.6.11.tgz", + "integrity": "sha512-INJ7ULu+Kj4UgqbcqE8Q+QpMw55xFf9kkyLBHJFk0ihjJpAV4glialRfqHE7k4KX2BWYPQfpILwhwR14x2EiRQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -25263,7 +26537,9 @@ } }, "rc-progress": { - "version": "3.2.4", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/rc-progress/-/rc-progress-3.3.3.tgz", + "integrity": "sha512-MDVNVHzGanYtRy2KKraEaWeZLri2ZHWIRyaE1a9MQ2MuJ09m+Wxj5cfcaoaR6z5iRpHpA59YeUxAlpML8N4PJw==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", @@ -25280,6 +26556,8 @@ }, "rc-resize-observer": { "version": "1.2.0", + "resolved": "https://registry.npmjs.org/rc-resize-observer/-/rc-resize-observer-1.2.0.tgz", + "integrity": "sha512-6W+UzT3PyDM0wVCEHfoW3qTHPTvbdSgiA43buiy8PzmeMnfgnDeb9NjdimMXMl3/TcrvvWl5RRVdp+NqcR47pQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", @@ -25297,7 +26575,9 @@ } }, "rc-select": { - "version": "14.1.2", + "version": "14.1.13", + "resolved": "https://registry.npmjs.org/rc-select/-/rc-select-14.1.13.tgz", + "integrity": "sha512-WMEsC3gTwA1dbzWOdVIXDmWyidYNLq68AwvvUlRROw790uGUly0/vmqDozXrIr0QvN/A3CEULx12o+WtLCAefg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", @@ -25335,67 +26615,83 @@ } }, "rc-table": { - "version": "7.24.1", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/rc-table/-/rc-table-7.26.0.tgz", + "integrity": "sha512-0cD8e6S+DTGAt5nBZQIPFYEaIukn17sfa5uFL98faHlH/whZzD8ii3dbFL4wmUDEL4BLybhYop+QUfZJ4CPvNQ==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-resize-observer": "^1.1.0", - "rc-util": "^5.14.0", + "rc-util": "^5.22.5", "shallowequal": "^1.1.0" } }, "rc-tabs": { - "version": "11.13.0", + "version": "12.2.1", + "resolved": "https://registry.npmjs.org/rc-tabs/-/rc-tabs-12.2.1.tgz", + "integrity": "sha512-09pVv4kN8VFqp6THceEmxOW8PAShQC08hrroeVYP4Y8YBFaP1PIWdyFL01czcbyz5YZFj9flZ7aljMaAl0jLVg==", "requires": { "@babel/runtime": "^7.11.2", "classnames": "2.x", - "rc-dropdown": "~3.5.0", - "rc-menu": "~9.5.1", + "rc-dropdown": "~4.0.0", + "rc-menu": "~9.6.0", + "rc-motion": "^2.6.2", "rc-resize-observer": "^1.0.0", "rc-util": "^5.5.0" } }, "rc-textarea": { - "version": "0.3.7", + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/rc-textarea/-/rc-textarea-0.4.6.tgz", + "integrity": "sha512-HEKCu8nouXXayqYelQnhQm8fdH7v92pAQvfVCz+jhIPv2PHTyBxVrmoZJMn3B8cU+wdyuvRGkshngO3/TzBn4w==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", "rc-resize-observer": "^1.0.0", - "rc-util": "^5.7.0", + "rc-util": "^5.24.4", "shallowequal": "^1.1.0" } }, "rc-tooltip": { - "version": "5.1.1", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/rc-tooltip/-/rc-tooltip-5.2.2.tgz", + "integrity": "sha512-jtQzU/18S6EI3lhSGoDYhPqNpWajMtS5VV/ld1LwyfrDByQpYmw/LW6U7oFXXLukjfDHQ7Ju705A82PRNFWYhg==", "requires": { "@babel/runtime": "^7.11.2", + "classnames": "^2.3.1", "rc-trigger": "^5.0.0" } }, "rc-tree": { - "version": "5.5.0", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/rc-tree/-/rc-tree-5.7.0.tgz", + "integrity": "sha512-F+Ewkv/UcutshnVBMISP+lPdHDlcsL+YH/MQDVWbk+QdkfID7vXiwrHMEZn31+2Rbbm21z/HPceGS8PXGMmnQg==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-util": "^5.16.1", - "rc-virtual-list": "^3.4.2" + "rc-virtual-list": "^3.4.8" } }, "rc-tree-select": { - "version": "5.3.0", + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/rc-tree-select/-/rc-tree-select-5.5.3.tgz", + "integrity": "sha512-gv8KyC6J7f9e50OkGk1ibF7v8vL+iaBnA8Ep/EVlMma2/tGdBQXO9xIvPjX8eQrZL5PjoeTUndNPM3cY3721ng==", "requires": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-select": "~14.1.0", - "rc-tree": "~5.5.0", + "rc-tree": "~5.7.0", "rc-util": "^5.16.1" } }, "rc-trigger": { - "version": "5.2.18", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/rc-trigger/-/rc-trigger-5.3.3.tgz", + "integrity": "sha512-IC4nuTSAME7RJSgwvHCNDQrIzhvGMKf6NDu5veX+zk1MG7i1UnwTWWthcP9WHw3+FZfP3oZGvkrHFPu/EGkFKw==", "requires": { - "@babel/runtime": "^7.11.2", + "@babel/runtime": "^7.18.3", "classnames": "^2.2.6", "rc-align": "^4.0.0", "rc-motion": "^2.0.0", @@ -25411,16 +26707,21 @@ } }, "rc-util": { - "version": "5.21.2", + "version": "5.24.4", + "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.24.4.tgz", + "integrity": "sha512-2a4RQnycV9eV7lVZPEJ7QwJRPlZNc06J7CwcwZo4vIHr3PfUqtYgl1EkUV9ETAc6VRRi8XZOMFhYG63whlIC9Q==", "requires": { - "@babel/runtime": "^7.12.5", + "@babel/runtime": "^7.18.3", "react-is": "^16.12.0", "shallowequal": "^1.1.0" } }, "rc-virtual-list": { - "version": "3.4.7", + "version": "3.4.11", + "resolved": "https://registry.npmjs.org/rc-virtual-list/-/rc-virtual-list-3.4.11.tgz", + "integrity": "sha512-BvUUH60kkeTBPigN5F89HtGaA5jSP4y2aM6cJ4dk9Y42I9yY+h6i08wF6UKeDcxdfOU8j3I5HxkSS/xA77J3wA==", "requires": { + "@babel/runtime": "^7.20.0", "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.15.0" @@ -25556,6 +26857,15 @@ "version": "0.11.0", "dev": true }, + "react-resizable": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/react-resizable/-/react-resizable-3.0.4.tgz", + "integrity": "sha512-StnwmiESiamNzdRHbSSvA65b0ZQJ7eVQpPusrSmcpyGKzC0gojhtO62xxH6YOBmepk9dQTBi9yxidL3W4s3EBA==", + "requires": { + "prop-types": "15.x", + "react-draggable": "^4.0.3" + } + }, "react-router": { "version": "6.3.0", "requires": { @@ -25681,7 +26991,9 @@ } }, "regenerator-runtime": { - "version": "0.13.9" + "version": "0.13.10", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.10.tgz", + "integrity": "sha512-KepLsg4dU12hryUO7bp/axHAKvwGOCV0sGloQtpagJ12ai+ojVDqkeGSiRX1zlq+kjIMZ1t7gpze+26QqtdGqw==" }, "regenerator-transform": { "version": "0.15.0", @@ -26160,6 +27472,15 @@ "version": "1.2.0", "dev": true }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "requires": { + "kind-of": "^6.0.2" + } + }, "shallowequal": { "version": "1.1.0" }, @@ -26340,7 +27661,9 @@ "dev": true }, "string-convert": { - "version": "0.2.1" + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", + "integrity": "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==" }, "string-length": { "version": "4.0.2", @@ -26461,7 +27784,9 @@ } }, "supports-hyperlinks": { - "version": "2.2.0", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", "dev": true, "requires": { "has-flag": "^4.0.0", @@ -26539,6 +27864,24 @@ "version": "3.2.4", "dev": true }, + "synckit": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.4.tgz", + "integrity": "sha512-Dn2ZkzMdSX827QbowGbU/4yjWuvNaCoScLLoMo/yKbu+P4GBR6cRGKZH27k6a9bRzdqcyd1DE96pQtQ6uNkmyw==", + "dev": true, + "requires": { + "@pkgr/utils": "^2.3.1", + "tslib": "^2.4.0" + }, + "dependencies": { + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", + "dev": true + } + } + }, "tailwindcss": { "version": "3.0.24", "dev": true, @@ -26652,6 +27995,16 @@ "version": "1.1.0", "dev": true }, + "tiny-glob": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz", + "integrity": "sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==", + "dev": true, + "requires": { + "globalyzer": "0.1.0", + "globrex": "^0.1.2" + } + }, "tmpl": { "version": "1.0.5", "dev": true @@ -26700,6 +28053,44 @@ "version": "1.0.1", "dev": true }, + "ts-node": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", + "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", + "dev": true, + "peer": true, + "requires": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "dependencies": { + "acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "dev": true, + "peer": true + }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "peer": true + } + } + }, "tsconfig-paths": { "version": "3.14.1", "dev": true, @@ -26824,6 +28215,16 @@ "version": "1.2.0", "dev": true }, + "update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "dev": true, + "requires": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + } + }, "uri-js": { "version": "4.4.1", "dev": true, @@ -26861,6 +28262,13 @@ "version": "2.3.0", "dev": true }, + "v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "peer": true + }, "v8-to-istanbul": { "version": "8.1.1", "dev": true, @@ -26880,6 +28288,43 @@ "version": "1.1.2", "dev": true }, + "vscode-json-languageservice": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", + "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "dev": true, + "requires": { + "jsonc-parser": "^3.0.0", + "vscode-languageserver-textdocument": "^1.0.3", + "vscode-languageserver-types": "^3.16.0", + "vscode-nls": "^5.0.0", + "vscode-uri": "^3.0.3" + } + }, + "vscode-languageserver-textdocument": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.7.tgz", + "integrity": "sha512-bFJH7UQxlXT8kKeyiyu41r22jCZXG8kuuVVA33OEJn1diWOZK5n8zBSPZFHVBOu8kXZ6h0LIRhf5UnCo61J4Hg==", + "dev": true + }, + "vscode-languageserver-types": { + "version": "3.17.2", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.2.tgz", + "integrity": "sha512-zHhCWatviizPIq9B7Vh9uvrH6x3sK8itC84HkamnBWoDFJtzBf7SWlpLCZUit72b3os45h6RWQNC9xHRDF8dRA==", + "dev": true + }, + "vscode-nls": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.2.0.tgz", + "integrity": "sha512-RAaHx7B14ZU04EU31pT+rKz2/zSl7xMsfIZuo8pd+KZO6PXtQmpevpq3vxvWNcrGbdmhM/rr5Uw5Mz+NBfhVng==", + "dev": true + }, + "vscode-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.6.tgz", + "integrity": "sha512-fmL7V1eiDBFRRnu+gfRWTzyPpNIHJTc4mWnFkwBUmO9U3KPgJAmTx7oxi2bl/Rh6HLdU7+4C9wlj0k2E4AdKFQ==", + "dev": true + }, "w3c-hr-time": { "version": "1.0.2", "dev": true, @@ -26926,6 +28371,8 @@ }, "webpack": { "version": "5.72.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.72.0.tgz", + "integrity": "sha512-qmSmbspI0Qo5ld49htys8GY9XhS9CGqFoHTsOVAnjBdg0Zn79y135R+k4IR4rKK6+eKaabMhJwiVB7xw0SJu5w==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.3", @@ -27074,6 +28521,16 @@ } } }, + "webpack-merge": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.8.0.tgz", + "integrity": "sha512-/SaI7xY0831XwP6kzuwhKWVKDP9t1QY1h65lAFLbZqMPIuYcD9QAW4u9STIbU9kaJbPBB/geU/gLr1wDjOhQ+Q==", + "dev": true, + "requires": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + } + }, "webpack-sources": { "version": "3.2.3", "dev": true @@ -27133,6 +28590,12 @@ "is-symbol": "^1.0.3" } }, + "wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==", + "dev": true + }, "word-wrap": { "version": "1.2.3", "dev": true @@ -27401,6 +28864,13 @@ "version": "20.2.9", "dev": true }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "peer": true + }, "yocto-queue": { "version": "0.1.0", "dev": true diff --git a/ui/package.json b/ui/package.json index 3a211df1f..c467c7b9e 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,20 +1,25 @@ { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0-rc2", "private": true, "dependencies": { + "@ant-design/icons": "^4.7.0", "@azure/msal-browser": "^2.24.0", "@azure/msal-react": "^1.4.0", - "antd": "^4.20.2", + "antd": "^4.23.6", "axios": "^0.27.2", + "classnames": "^2.3.2", "dagre": "^0.8.5", + "dayjs": "^1.11.5", "react": "^17.0.2", "react-dom": "^17.0.2", "react-flow-renderer": "^9.7.4", "react-query": "^3.38.0", + "react-resizable": "^3.0.4", "react-router-dom": "^6.3.0" }, "devDependencies": { + "@craco/craco": "^7.0.0-alpha.8", "@testing-library/jest-dom": "^5.16.3", "@testing-library/react": "^12.1.4", "@testing-library/user-event": "^13.5.0", @@ -23,25 +28,34 @@ "@types/node": "^16.11.26", "@types/react": "^17.0.43", "@types/react-dom": "^17.0.14", + "@types/react-resizable": "^3.0.3", "@typescript-eslint/eslint-plugin": "^5.30.7", "@typescript-eslint/parser": "^5.30.7", + "babel-plugin-import": "^1.13.5", + "craco-less": "^2.1.0-alpha.0", "eslint": "^8.20.0", "eslint-config-prettier": "^8.5.0", + "eslint-import-resolver-typescript": "^3.5.1", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-json": "^3.1.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.1", "lint-staged": "^13.0.3", "prettier": "2.7.1", "react-scripts": "5.0.0", "typescript": "^4.6.3", - "web-vitals": "^2.1.4" + "web-vitals": "^2.1.4", + "webpack": "^5.72.0" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test", + "start": "craco start", + "build": "craco build", + "test": "craco test", "eject": "react-scripts eject", "lint:fix": "npx eslint --fix --ext ts --ext tsx src/ ", - "format": "npx prettier --write src/**" + "format": "npx prettier --write src/**", + "lintStaged": "lint-staged" }, "browserslist": { "production": [ diff --git a/ui/public/favicon.ico b/ui/public/favicon.ico index a11777cc4..fc2f6ca0f 100644 Binary files a/ui/public/favicon.ico and b/ui/public/favicon.ico differ diff --git a/ui/public/index.html b/ui/public/index.html index 0050dcf77..d0bc57b87 100644 --- a/ui/public/index.html +++ b/ui/public/index.html @@ -9,7 +9,7 @@ name="description" content="Feathr Feature Store Web UI" /> - + Feathr Feature Store diff --git a/ui/public/logo192.png b/ui/public/logo192.png deleted file mode 100644 index fc44b0a37..000000000 Binary files a/ui/public/logo192.png and /dev/null differ diff --git a/ui/public/logo200.png b/ui/public/logo200.png new file mode 100644 index 000000000..254621fb0 Binary files /dev/null and b/ui/public/logo200.png differ diff --git a/ui/public/logo512.png b/ui/public/logo512.png deleted file mode 100644 index a4e47a654..000000000 Binary files a/ui/public/logo512.png and /dev/null differ diff --git a/ui/public/manifest.json b/ui/public/manifest.json index 50a99047f..f6d4ea50a 100644 --- a/ui/public/manifest.json +++ b/ui/public/manifest.json @@ -8,14 +8,9 @@ "type": "image/x-icon" }, { - "src": "logo192.png", + "src": "logo200.png", "type": "image/png", - "sizes": "192x192" - }, - { - "src": "logo512.png", - "type": "image/png", - "sizes": "512x512" + "sizes": "200x200" } ], "start_url": ".", diff --git a/ui/src/api/api.tsx b/ui/src/api/api.tsx index a95ab2bd5..6c8b6f665 100644 --- a/ui/src/api/api.tsx +++ b/ui/src/api/api.tsx @@ -38,14 +38,18 @@ export const fetchDataSource = async ( ) => { const axios = await authAxios(msalInstance); return axios - .get( + .get( `${getApiBaseUrl()}/projects/${project}/datasources/${dataSourceId}`, { params: { project: project, datasource: dataSourceId }, } ) .then((response) => { - return response.data; + if (response.data.message || response.data.detail) { + return Promise.reject(response.data.message || response.data.detail); + } else { + return response.data; + } }); }; @@ -109,33 +113,21 @@ export const fetchFeatureLineages = async (featureId: string) => { // Following are place-holder code export const createFeature = async (feature: Feature) => { const axios = await authAxios(msalInstance); - return axios - .post(`${getApiBaseUrl()}/features`, feature, { - headers: { "Content-Type": "application/json;" }, - params: {}, - }) - .then((response) => { - return response; - }) - .catch((error) => { - return error.response; - }); + return axios.post(`${getApiBaseUrl()}/features`, feature, { + headers: { "Content-Type": "application/json;" }, + params: {}, + }); }; -export const updateFeature = async (feature: Feature, id: string) => { +export const updateFeature = async (feature: Feature, id?: string) => { const axios = await authAxios(msalInstance); - feature.guid = id; - return await axios - .put(`${getApiBaseUrl()}/features/${id}`, feature, { - headers: { "Content-Type": "application/json;" }, - params: {}, - }) - .then((response) => { - return response; - }) - .catch((error) => { - return error.response; - }); + if (id) { + feature.guid = id; + } + return axios.put(`${getApiBaseUrl()}/features/${feature.guid}`, feature, { + headers: { "Content-Type": "application/json;" }, + params: {}, + }); }; export const listUserRole = async () => { @@ -245,6 +237,8 @@ export const authAxios = async (msalInstance: PublicClientApplication) => { if (error.response?.status === 403) { const detail = error.response.data.detail; window.location.href = "/responseErrors/403/" + detail; + } else { + return Promise.reject(error.response.data); } //TODO: handle other response errors } diff --git a/ui/src/app.tsx b/ui/src/app.tsx index 5984717f9..8b43cc70a 100644 --- a/ui/src/app.tsx +++ b/ui/src/app.tsx @@ -20,48 +20,56 @@ import RoleManagement from "./pages/management/roleManagement"; import Home from "./pages/home/home"; import Projects from "./pages/project/projects"; import { getMsalConfig } from "./utils/utils"; +import Footer from "@/components/footer"; const queryClient = new QueryClient(); const msalClient = getMsalConfig(); + const App = () => { return ( - +
- - } /> - } /> - } /> - } /> - } /> - } /> - } - /> - } - /> - } - /> - } /> - } /> - } /> - } /> - } - /> - + + + } /> + } /> + } /> + } /> + } /> + } /> + } + /> + } + /> + } + /> + } /> + } /> + } /> + } + /> + } + /> + + +