Change to pypi detect-secrets (#1377)

* Change to pypi detect-secrets Signed-off-by: elronbandel <[email protected]> * Delete unnecassery req files Signed-off-by: elronbandel <[email protected]> * SImplify docs Signed-off-by: elronbandel <[email protected]> --------- Signed-off-by: elronbandel <[email protected]>
IBM · Nov 19, 2024 · 48538a9 · 48538a9
1 parent e4732c4
commit 48538a9
Show file tree

Hide file tree

Showing 12 changed files with 213 additions and 108 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -22,20 +22,12 @@ repos:
       # Run the formatter
       - id: ruff-format
 
-  - repo: https://github.com/ibm/detect-secrets
-    # If you desire to use a specific version of detect-secrets, you can replace `master` with other git revisions such as branch, tag or commit sha.
-    # You are encouraged to use static refs such as tags, instead of branch name
-    #
-    # Running "pre-commit autoupdate" automatically updates rev to latest tag
-    rev: 0.13.1+ibm.61.dss
+  - repo: https://github.com/Yelp/detect-secrets
+    rev: v1.5.0
     hooks:
-      - id: detect-secrets # pragma: whitelist secret
-        # Add options for detect-secrets-hook binary. You can run `detect-secrets-hook --help` to list out all possible options.
-        # You may also run `pre-commit run detect-secrets` to preview the scan result.
-        # when "--baseline" without "--use-all-plugins", pre-commit scan with just plugins in baseline file
-        # when "--baseline" with "--use-all-plugins", pre-commit scan with all available plugins
-        # add "--fail-on-unaudited" to fail pre-commit for unaudited potential secrets
-        args: [--baseline, utils/.secrets.baseline, --use-all-plugins, --fail-on-unaudited]
+      - id: detect-secrets
+        args: [--baseline, utils/.secrets.baseline]
+        exclude: utils/.secrets.baseline
 
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unitxt"
-dynamic = ["version", "optional-dependencies"]
+dynamic = ["version"]
 description = "Load any mixture of text to text data in one line of code"
 authors = [
     { name = "IBM Research", email = "[email protected]" }
@@ -42,7 +42,95 @@ unitxt = ["catalog/**/*.json", "ui/banner.png"]
 
 [tool.setuptools.dynamic]
 version = {attr = "unitxt.version.version"}
-optional-dependencies = {dev = { file = ["utils/requirements/dev.txt"] }, docs = { file = ["utils/requirements/docs.txt"] }, helm = { file = ["utils/requirements/helm.txt"] }, service = { file = ["utils/requirements/service.txt"] }, tests = { file = ["utils/requirements/tests.txt"] }, ui = { file = ["utils/requirements/ui.txt"] }, watsonx = { file = ["utils/requirements/watsonx.txt"] }, "inference-tests" = { file = ["utils/requirements/inference-tests.txt"] }, all = { file = ["utils/requirements/all.txt"] }}
+
+[project.optional-dependencies]
+dev = [
+    "ruff",
+    "pre-commit",
+    "detect-secrets",
+    "tomli",
+    "codespell",
+    "fuzzywuzzy",
+    "httpretty"
+]
+docs = [
+    "sphinx_rtd_theme",
+    "piccolo_theme",
+    "sphinxext-opengraph",
+    "datasets",
+    "evaluate",
+    "nltk",
+    "sacrebleu",
+    "absl-py",
+    "rouge_score",
+    "scikit-learn",
+    "jiwer",
+    "editdistance",
+    "fuzzywuzzy"
+]
+helm = [
+    "crfm-helm[unitxt]>=0.5.3"
+]
+service = [
+    "torch==1.12.1",
+    "fastapi==0.109.0",
+    "uvicorn[standard]==0.27.0.post1",
+    "python-jose[cryptography]==3.3.0",
+    "transformers"
+]
+tests = [
+    "bert_score",
+    "transformers",
+    "sentence_transformers",
+    "ibm-cos-sdk",
+    "kaggle==1.6.14",
+    "opendatasets",
+    "httpretty~=1.1.4",
+    "editdistance",
+    "rouge-score",
+    "nltk",
+    "mecab-python3",
+    "sacrebleu[ko]",
+    "scikit-learn",
+    "jiwer",
+    "conllu",
+    "llama-index-core",
+    "llama-index-llms-openai",
+    "pytrec-eval",
+    "SentencePiece",
+    "fuzzywuzzy",
+    "openai",
+    "ibm-generative-ai",
+    "bs4",
+    "tenacity==8.3.0",
+    "accelerate",
+    "spacy"
+]
+ui = [
+    "gradio",
+    "transformers"
+]
+watsonx = [
+    "ibm-watsonx-ai==1.1.14"
+]
+inference-tests = [
+  "litellm==v1.52.9",
+  "tenacity",
+  "diskcache",
+  "numpy==1.26.4"
+]
+
+# Combine all extras by referencing other groups
+all = [
+    "unitxt[base]",
+    "unitxt[dev]",
+    "unitxt[docs]",
+    "unitxt[helm]",
+    "unitxt[service]",
+    "unitxt[tests]",
+    "unitxt[ui]",
+    "unitxt[watsonx]"
+]
 
 [project.scripts]
 unitxt-explore = "unitxt.ui:launch"

diff --git a/utils/.secrets.baseline b/utils/.secrets.baseline
@@ -1,65 +1,72 @@
 {
-  "exclude": {
-    "files": "^utils/.secrets.baseline$",
-    "lines": null
-  },
-  "generated_at": "2024-11-10T18:29:49Z",
+  "version": "1.5.0",
   "plugins_used": [
     {
-      "name": "AWSKeyDetector"
+      "name": "ArtifactoryDetector"
     },
     {
-      "name": "ArtifactoryDetector"
+      "name": "AWSKeyDetector"
     },
     {
       "name": "AzureStorageKeyDetector"
     },
     {
-      "base64_limit": 4.5,
-      "name": "Base64HighEntropyString"
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
     },
     {
       "name": "BasicAuthDetector"
     },
-    {
-      "name": "BoxDetector"
-    },
     {
       "name": "CloudantDetector"
     },
     {
-      "ghe_instance": "github.ibm.com",
-      "name": "GheDetector"
+      "name": "DiscordBotTokenDetector"
     },
     {
       "name": "GitHubTokenDetector"
     },
     {
-      "hex_limit": 3,
-      "name": "HexHighEntropyString"
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
     },
     {
       "name": "IbmCloudIamDetector"
     },
     {
       "name": "IbmCosHmacDetector"
     },
+    {
+      "name": "IPPublicDetector"
+    },
     {
       "name": "JwtTokenDetector"
     },
     {
-      "keyword_exclude": null,
-      "name": "KeywordDetector"
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
     },
     {
       "name": "MailchimpDetector"
     },
     {
       "name": "NpmDetector"
     },
+    {
+      "name": "OpenAIDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
     {
       "name": "SlackDetector"
     },
@@ -72,25 +79,110 @@
     {
       "name": "StripeDetector"
     },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
     {
       "name": "TwilioKeyDetector"
     }
   ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": "utils/.secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
   "results": {
+    "src/unitxt/inference.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/unitxt/inference.py",
+        "hashed_secret": "aa6cd2a77de22303be80e1f632195d62d211a729",
+        "is_verified": false,
+        "line_number": 562,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "src/unitxt/inference.py",
+        "hashed_secret": "c8f16a194efc59559549c7bd69f7bea038742e79",
+        "is_verified": false,
+        "line_number": 893,
+        "is_secret": false
+      }
+    ],
+    "src/unitxt/loaders.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "src/unitxt/loaders.py",
+        "hashed_secret": "840268f77a57d5553add023cfa8a4d1535f49742",
+        "is_verified": false,
+        "line_number": 483,
+        "is_secret": false
+      }
+    ],
     "src/unitxt/metrics.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "src/unitxt/metrics.py",
         "hashed_secret": "fa172616e9af3d2a24b5597f264eab963fe76889",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2083,
-        "type": "Hex High Entropy String",
-        "verified_result": null
+        "is_secret": false
+      }
+    ],
+    "tests/library/test_loaders.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/library/test_loaders.py",
+        "hashed_secret": "8d814baafe5d8412572dc520dcab83f60ce1375c",
+        "is_verified": false,
+        "line_number": 113,
+        "is_secret": false
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/library/test_loaders.py",
+        "hashed_secret": "42a472ac88cd8d43a2c5ae0bd0bdf4626cdaba31",
+        "is_verified": false,
+        "line_number": 123,
+        "is_secret": false
       }
     ]
   },
-  "version": "0.13.1+ibm.61.dss",
-  "word_list": {
-    "file": null,
-    "hash": null
-  }
+  "generated_at": "2024-11-19T21:32:11Z"
 }
diff --git a/utils/requirements/all.txt b/utils/requirements/all.txt
diff --git a/utils/requirements/dev.txt b/utils/requirements/dev.txt
diff --git a/utils/requirements/docs.txt b/utils/requirements/docs.txt
diff --git a/utils/requirements/helm.txt b/utils/requirements/helm.txt
diff --git a/utils/requirements/inference-tests.txt b/utils/requirements/inference-tests.txt
diff --git a/utils/requirements/service.txt b/utils/requirements/service.txt