From a8b59e7b6655f4ca24b007a0b37915a6d6b44663 Mon Sep 17 00:00:00 2001 From: Steve Yonkeu Date: Sat, 26 Oct 2024 04:47:56 +0100 Subject: [PATCH] initial setup and structure --- .gitignore | 163 ++++++++++++ CODE_OF_CONDUCT.md | 3 + IaaC/README.md | 1 + LICENSE | 21 ++ README.md | 53 ++++ SECURITY.md | 3 + analytics/README.md | 87 +++++++ analytics/aws_cloudwatch_analytics.py | 29 +++ analytics/azure_monitor_analytics.py | 30 +++ analytics/google_analytics.py | 28 +++ analytics/mixpanel_analytics.py | 18 ++ analytics/sentry_analytics.py | 21 ++ apis/README.md | 115 +++++++++ apis/graphql/flask_ariadne.py | 31 +++ apis/graphql/graphene_django/main.py | 11 + apis/grpc/grpc_python.py | 22 ++ apis/grpc/service.proto | 13 + apis/grpc/service_pb2.py | 40 +++ apis/grpc/service_pb2_grpc.py | 97 ++++++++ apis/mqtt/mqtt.py | 25 ++ apis/rest_apis/django_ninja_rest.py | 8 + apis/rest_apis/django_rest.py | 17 ++ apis/rest_apis/flask_rest.py | 16 ++ apis/soap/zeep_python.py | 28 +++ apis/web_sockets/django_channels.py | 7 + apis/web_sockets/flask_sockets.py | 12 + caching/README.md | 174 +++++++++++++ caching/db_cache.py | 32 +++ caching/disk_cache.py | 15 ++ caching/fifo_cache.py | 27 ++ caching/lfu_cache.py | 19 ++ caching/lifo_cache.py | 27 ++ caching/lru_cache.py | 10 + caching/memcached.py | 18 ++ caching/redis_cache/main.py | 28 +++ caching/redis_cache/redis.conf | 1 + clean_architecture/README.md | 1 + database/README.md | 192 ++++++++++++++ deployments/README.md | 1 + docker/README.md | 1 + emails/README.md | 127 ++++++++++ emails/aws_ses_email.py | 65 +++++ emails/azure_email.py | 57 +++++ emails/mailersend_email.py | 60 +++++ emails/mailgun_email.py | 43 ++++ emails/sendgrid_email.py | 61 +++++ emails/smtp_email.py | 53 ++++ env_secrets/README.md | 1 + logging/README.md | 109 ++++++++ logging/aws_cloudwatch_logging.py | 21 ++ logging/azure_monitor_logging.py | 27 ++ logging/basic_logging.py | 13 + logging/gcp_logging.py | 53 ++++ logging/structured_logging.py | 24 ++ machine_learning/README.md | 1 + monitoring/README.md | 130 ++++++++++ monitoring/checkmk/checkmk_setup.md | 0 monitoring/datadog/datadog_example.py | 5 + monitoring/datadog/datadog_setup.md | 31 +++ .../elasticsearch/elastic_log_integration.py | 0 monitoring/grafana/grafana_setup.md | 21 ++ monitoring/influxdb/influxdb_integration.py | 0 .../prometheus/prometheus_client_example.py | 13 + monitoring/prometheus/prometheus_details.md | 50 ++++ monitoring/sentry/sentry_integration.py | 25 ++ monitoring/sentry/sentry_setup.md | 33 +++ monitoring/zabbix/zabbix_api_example.py | 0 payments/README.md | 235 ++++++++++++++++++ payments/braintree_payment.py | 46 ++++ payments/btcpay_payment.py | 46 ++++ payments/chargebee_payment.py | 100 ++++++++ payments/getlago_payment.py | 48 ++++ payments/mpesa_payment.py | 55 ++++ payments/mtn_momo_payment.py | 53 ++++ payments/orange_money_payment.py | 65 +++++ payments/paypal_payment.py | 50 ++++ payments/paystack_payment.py | 45 ++++ payments/razorpay_payment.py | 42 ++++ payments/squareup_payment.py | 48 ++++ payments/stripe_payment.py | 41 +++ queueing/README.md | 1 + scaling/README.md | 1 + security/README.md | 202 +++++++++++++++ security/aes_encryption.py | 26 ++ security/apache_setup.md | 64 +++++ security/bcrypt_passwords.py | 16 ++ security/cors.py | 24 ++ security/csrf.py | 24 ++ security/flask_headers.py | 12 + security/flask_rate_limiting.py | 22 ++ security/jwt_auth.py | 31 +++ security/otp.py | 15 ++ security/passkeys.py | 42 ++++ security/rbac_abac.py | 17 ++ security/tls_nginx_security.conf | 11 + sms/README.md | 109 ++++++++ sms/africastalking_sms.py | 37 +++ sms/aws_sms.py | 44 ++++ sms/azure_sms.py | 38 +++ sms/nexmo_sms.py | 42 ++++ sms/twilio_sms.py | 36 +++ storage/README.md | 95 +++++++ storage/aws_s3_storage.py | 77 ++++++ storage/azure_blob_storage.py | 75 ++++++ storage/gcp_storage.py | 81 ++++++ storage/google_drive_storage.py | 88 +++++++ 106 files changed, 4572 insertions(+) create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 IaaC/README.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 analytics/README.md create mode 100644 analytics/aws_cloudwatch_analytics.py create mode 100644 analytics/azure_monitor_analytics.py create mode 100644 analytics/google_analytics.py create mode 100644 analytics/mixpanel_analytics.py create mode 100644 analytics/sentry_analytics.py create mode 100644 apis/README.md create mode 100644 apis/graphql/flask_ariadne.py create mode 100644 apis/graphql/graphene_django/main.py create mode 100644 apis/grpc/grpc_python.py create mode 100644 apis/grpc/service.proto create mode 100644 apis/grpc/service_pb2.py create mode 100644 apis/grpc/service_pb2_grpc.py create mode 100644 apis/mqtt/mqtt.py create mode 100644 apis/rest_apis/django_ninja_rest.py create mode 100644 apis/rest_apis/django_rest.py create mode 100644 apis/rest_apis/flask_rest.py create mode 100644 apis/soap/zeep_python.py create mode 100644 apis/web_sockets/django_channels.py create mode 100644 apis/web_sockets/flask_sockets.py create mode 100644 caching/README.md create mode 100644 caching/db_cache.py create mode 100644 caching/disk_cache.py create mode 100644 caching/fifo_cache.py create mode 100644 caching/lfu_cache.py create mode 100644 caching/lifo_cache.py create mode 100644 caching/lru_cache.py create mode 100644 caching/memcached.py create mode 100644 caching/redis_cache/main.py create mode 100644 caching/redis_cache/redis.conf create mode 100644 clean_architecture/README.md create mode 100644 database/README.md create mode 100644 deployments/README.md create mode 100644 docker/README.md create mode 100644 emails/README.md create mode 100644 emails/aws_ses_email.py create mode 100644 emails/azure_email.py create mode 100644 emails/mailersend_email.py create mode 100644 emails/mailgun_email.py create mode 100644 emails/sendgrid_email.py create mode 100644 emails/smtp_email.py create mode 100644 env_secrets/README.md create mode 100644 logging/README.md create mode 100644 logging/aws_cloudwatch_logging.py create mode 100644 logging/azure_monitor_logging.py create mode 100644 logging/basic_logging.py create mode 100644 logging/gcp_logging.py create mode 100644 logging/structured_logging.py create mode 100644 machine_learning/README.md create mode 100644 monitoring/README.md create mode 100644 monitoring/checkmk/checkmk_setup.md create mode 100644 monitoring/datadog/datadog_example.py create mode 100644 monitoring/datadog/datadog_setup.md create mode 100644 monitoring/elasticsearch/elastic_log_integration.py create mode 100644 monitoring/grafana/grafana_setup.md create mode 100644 monitoring/influxdb/influxdb_integration.py create mode 100644 monitoring/prometheus/prometheus_client_example.py create mode 100644 monitoring/prometheus/prometheus_details.md create mode 100644 monitoring/sentry/sentry_integration.py create mode 100644 monitoring/sentry/sentry_setup.md create mode 100644 monitoring/zabbix/zabbix_api_example.py create mode 100644 payments/README.md create mode 100644 payments/braintree_payment.py create mode 100644 payments/btcpay_payment.py create mode 100644 payments/chargebee_payment.py create mode 100644 payments/getlago_payment.py create mode 100644 payments/mpesa_payment.py create mode 100644 payments/mtn_momo_payment.py create mode 100644 payments/orange_money_payment.py create mode 100644 payments/paypal_payment.py create mode 100644 payments/paystack_payment.py create mode 100644 payments/razorpay_payment.py create mode 100644 payments/squareup_payment.py create mode 100644 payments/stripe_payment.py create mode 100644 queueing/README.md create mode 100644 scaling/README.md create mode 100644 security/README.md create mode 100644 security/aes_encryption.py create mode 100644 security/apache_setup.md create mode 100644 security/bcrypt_passwords.py create mode 100644 security/cors.py create mode 100644 security/csrf.py create mode 100644 security/flask_headers.py create mode 100644 security/flask_rate_limiting.py create mode 100644 security/jwt_auth.py create mode 100644 security/otp.py create mode 100644 security/passkeys.py create mode 100644 security/rbac_abac.py create mode 100644 security/tls_nginx_security.conf create mode 100644 sms/README.md create mode 100644 sms/africastalking_sms.py create mode 100644 sms/aws_sms.py create mode 100644 sms/azure_sms.py create mode 100644 sms/nexmo_sms.py create mode 100644 sms/twilio_sms.py create mode 100644 storage/README.md create mode 100644 storage/aws_s3_storage.py create mode 100644 storage/azure_blob_storage.py create mode 100644 storage/gcp_storage.py create mode 100644 storage/google_drive_storage.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec21109 --- /dev/null +++ b/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ +.history/ diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f57cb04 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,3 @@ +What can we say? + +JUST BE NICE TO EACH OTHER! diff --git a/IaaC/README.md b/IaaC/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/IaaC/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..709e7c5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Steve Yonkeu (@yokwejuste) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..929c39a --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ + +

+

🚀BACKEND SECRETS

+

+ +

+ Typing SVG +

+ + +**Backend Secrets** is a comprehensive collection of Python implementations covering various backend functionalities, organized into distinct modules. This repository is designed to serve as a one-stop resource for backend developers looking to streamline common tasks, enhance security, and optimize performance in their applications. Each folder addresses a specific area of backend development, from caching and authentication to machine learning and server scaling. + +## Directory Structure + +- **analytics**: Tools and libraries for tracking, analyzing, and visualizing application usage and metrics. +- **apis**: Examples for implementing various API types, including REST, GraphQL, gRPC, and more. +- **caching**: Different caching mechanisms like Redis, LRU, LFU, and file-based caching to improve application performance. +- **clean_architecture**: Design principles and patterns for structuring Python applications in a maintainable and scalable way. +- **database**: Implementations for connecting and interacting with multiple database types, including MySQL, PostgreSQL, MongoDB, Redis, and more. +- **deployments**: Deployment scripts and tools for setting up and scaling applications across environments. +- **docker**: Docker configurations to containerize services, making deployments easier and more consistent. +- **emails**: Code examples for integrating with popular email services like AWS SES, Mailgun, and SendGrid. +- **env_secrets**: Techniques for securely managing environment variables and sensitive data in Python applications. +- **IaC (Infrastructure as Code)**: Scripts for automating infrastructure provisioning and management using tools like Terraform and CloudFormation. +- **logging**: Logging solutions for tracking application events, errors, and performance metrics. +- **machine_learning**: Modules for integrating machine learning models and handling tasks related to data processing and predictions. +- **monitoring**: Tools for tracking system and application performance, including Prometheus, Grafana, and Datadog integrations. +- **payments**: Implementations for handling payments with various providers such as Stripe, PayPal, and local options like M-Pesa. +- **queueing**: Message queue implementations with RabbitMQ, Redis Queue, and Celery for handling background tasks. +- **scaling**: Techniques and tools to scale Python applications effectively, including load balancing and horizontal scaling. +- **security**: Security features such as encryption, authentication, rate limiting, and best practices for securing APIs. +- **sms**: Code for sending SMS using providers like Twilio, Nexmo, AWS SNS, and others. +- **storage**: Integrations with cloud storage providers like AWS S3, Google Cloud Storage, and Azure Blob. + +## Getting Started + +1. **Clone the repository**: + ```bash + git clone https://github.com//backend-secrets.git + cd backend-secrets + ``` + +2. **Install Dependencies**: Each folder contains instructions for installing necessary libraries. You can also create a virtual environment and install dependencies separately for each module. + +3. **Run Examples**: Navigate to the desired module and follow the instructions in its README to run specific examples. + +## Contributing + +Contributions are welcome! If you have a feature to add, a bug to fix, or an improvement suggestion, please submit a pull request with a description of the changes. + +## License + +This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..8a7ddf9 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security + +As for the security, I personally think we will opt into the best practices for securing our backend services. Please if anyone has any suggestions kindly raise an issue or email me at `yokwejuste@gmail.com`. diff --git a/analytics/README.md b/analytics/README.md new file mode 100644 index 0000000..766cf53 --- /dev/null +++ b/analytics/README.md @@ -0,0 +1,87 @@ +# Analytics + +This folder contains Python implementations for integrating various analytics tools into backend applications. Each script provides functions to send events, track metrics, and monitor application usage using different analytics platforms. + +## AWS CloudWatch + +AWS CloudWatch is a monitoring and observability service from AWS that allows you to collect and track metrics, set alarms, and automatically respond to changes in AWS resources. + +- **File**: `aws_cloudwatch_analytics.py` +- **Setup**: + 1. Install the AWS SDK: `pip install boto3`. + 2. Configure AWS credentials with access to CloudWatch. + +- **Example Usage**: + ```python + from aws_cloudwatch_analytics import put_custom_metric + put_custom_metric("MyAppNamespace", "UserLogins", 1) + ``` + +## Azure Monitor + +Azure Monitor provides full-stack monitoring for applications, infrastructure, and network. It includes features like metrics tracking, alerting, and visualization. + +- **File**: `azure_monitor_analytics.py` +- **Setup**: + 1. Install Azure SDK: `pip install azure-monitor-query azure-identity`. + 2. Configure Azure credentials (client ID, tenant ID, and client secret) for authentication. + +- **Example Usage**: + ```python + from azure_monitor_analytics import query_metrics + query_metrics("") + ``` + +## Google Analytics + +Google Analytics allows you to track website and application usage using events, page views, and other metrics through the Measurement Protocol. + +- **File**: `google_analytics.py` +- **Setup**: + 1. No additional installation required, but you’ll need a Google Analytics tracking ID. + +- **Example Usage**: + ```python + from google_analytics import send_event + send_event("category", "action", "label", 10) + ``` + +## Mixpanel + +Mixpanel is an advanced analytics tool focused on tracking user interactions and product usage to provide insights into how users engage with your application. + +- **File**: `mixpanel_analytics.py` +- **Setup**: + 1. Install the Mixpanel client: `pip install mixpanel`. + +- **Example Usage**: + ```python + from mixpanel_analytics import track_event + track_event("UserSignUp", "user_123", {"plan": "premium"}) + ``` + +## Sentry + +Sentry is primarily an error tracking tool, but it also provides application performance monitoring. It’s ideal for capturing and reporting exceptions in real-time. + +- **File**: `sentry_analytics.py` +- **Setup**: + 1. Install the Sentry SDK: `pip install sentry-sdk`. + 2. Initialize Sentry with your project’s DSN. + +- **Example Usage**: + ```python + from sentry_analytics import log_error + log_error() + ``` + +## How to Use + +1. Clone the repository and navigate to the `analytics` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your credentials for each service (e.g., AWS access keys, Google Analytics tracking ID). +4. Run the scripts to integrate analytics tracking into your application. + +## Contributing + +Feel free to contribute by adding more analytics tools or improving existing ones. Please submit a pull request with a detailed description of your changes. \ No newline at end of file diff --git a/analytics/aws_cloudwatch_analytics.py b/analytics/aws_cloudwatch_analytics.py new file mode 100644 index 0000000..fbfb598 --- /dev/null +++ b/analytics/aws_cloudwatch_analytics.py @@ -0,0 +1,29 @@ +import os + +import boto3 +from dotenv import load_dotenv + +load_dotenv() + +AWS_REGION = os.getenv('AWS_REGION') + +# Initialize CloudWatch client +cloudwatch = boto3.client('cloudwatch', region_name=AWS_REGION) + + +def put_custom_metric(namespace, metric_name, value): + response = cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=[ + { + 'MetricName': metric_name, + 'Value': value, + 'Unit': 'Count' + }, + ] + ) + print(f"Metric sent: {metric_name} - Value: {value}") + + +# Example usage +put_custom_metric("MyAppNamespace", "UserLogins", 1) diff --git a/analytics/azure_monitor_analytics.py b/analytics/azure_monitor_analytics.py new file mode 100644 index 0000000..387a2d0 --- /dev/null +++ b/analytics/azure_monitor_analytics.py @@ -0,0 +1,30 @@ +from azure.monitor.query import MetricsQueryClient +from azure.identity import DefaultAzureCredential +from dotenv import load_dotenv +import os + +AZURE_CLIENT_ID = "AZURE_CLIENT_ID" +AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET" +AZURE_TENANT_ID = "AZURE_TENANT_ID" + +load_dotenv() + +credential = DefaultAzureCredential( + client_id=os.getenv(AZURE_CLIENT_ID), + client_secret=os.getenv(AZURE_CLIENT_SECRET), + tenant_id=os.getenv(AZURE_TENANT_ID) +) + +client = MetricsQueryClient(credential) + +def query_metrics(azure_resource_id): + response = client.query(azure_resource_id, metric_names=["UserLogins"], timespan="PT1H") + for metric in response.metrics: + print(f"Metric name: {metric.name}") + for time_series in metric.timeseries: + for data in time_series.data: + print(f"Timestamp: {data.timestamp}, Value: {data.total}") + +# Example usage +resource_id = "/subscriptions//resourceGroups//providers/Microsoft.Web/sites/" +query_metrics(resource_id) diff --git a/analytics/google_analytics.py b/analytics/google_analytics.py new file mode 100644 index 0000000..df93762 --- /dev/null +++ b/analytics/google_analytics.py @@ -0,0 +1,28 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +GA_TRACKING_ID = os.getenv('GA_TRACKING_ID') +GA_API_ENDPOINT = "https://www.google-analytics.com/collect" + + +def send_event(category, action, label=None, value=None): + payload = { + 'v': '1', + 'tid': GA_TRACKING_ID, + 'cid': '555', + 't': 'event', + 'ec': category, + 'ea': action, + 'el': label, # (optional) + 'ev': value, # (optional) + } + response = requests.post(GA_API_ENDPOINT, data=payload) + print(f"Event sent. Status code: {response.status_code}") + + +# Example usage +send_event("test_category", "test_action", "test_label", 10) diff --git a/analytics/mixpanel_analytics.py b/analytics/mixpanel_analytics.py new file mode 100644 index 0000000..7927786 --- /dev/null +++ b/analytics/mixpanel_analytics.py @@ -0,0 +1,18 @@ +import os + +from dotenv import load_dotenv +from mixpanel import Mixpanel + +load_dotenv() + +MIXPANEL_PROJECT_TOKEN = os.getenv('MIXPANEL_PROJECT_TOKEN') +mp = Mixpanel(MIXPANEL_PROJECT_TOKEN) + + +def track_event(event_name, distinct_id, properties=None): + mp.track(distinct_id, event_name, properties) + print(f"Event '{event_name}' tracked for user {distinct_id}") + + +# Example usage +track_event("UserSignUp", "user_123", {"plan": "premium"}) diff --git a/analytics/sentry_analytics.py b/analytics/sentry_analytics.py new file mode 100644 index 0000000..da92fe6 --- /dev/null +++ b/analytics/sentry_analytics.py @@ -0,0 +1,21 @@ +import os + +import sentry_sdk +from dotenv import load_dotenv + +load_dotenv() + +SENTRY_DSN = os.getenv('SENTRY_DSN') +sentry_sdk.init(SENTRY_DSN, traces_sample_rate=1.0) + + +def log_error(): + try: + 1 / 0 # Example error + except ZeroDivisionError as e: + sentry_sdk.capture_exception(e) + print("Error captured by Sentry.") + + +# Example usage +log_error() diff --git a/apis/README.md b/apis/README.md new file mode 100644 index 0000000..4c33a22 --- /dev/null +++ b/apis/README.md @@ -0,0 +1,115 @@ +# APIs + +This folder contains Python implementations for various API types and frameworks, including **GraphQL**, **gRPC**, **MQTT**, **REST APIs**, **SOAP**, and **WebSockets**. Each folder includes examples and code for setting up these APIs with popular Python libraries and frameworks. + +## Contents + +- [GraphQL](#graphql) +- [gRPC](#grpc) +- [MQTT](#mqtt) +- [REST APIs](#rest-apis) +- [SOAP](#soap) +- [WebSockets](#websockets) + +--- + +### GraphQL + +GraphQL APIs enable clients to request exactly the data they need, improving efficiency and flexibility. + +- **Graphene with Django** (`graphene_django.py`): Example of setting up a GraphQL API using the Graphene library with Django. +- **Flask with Ariadne** (`flask_ariadne.py`): Example of using Ariadne with Flask for schema-first GraphQL development. + +**Usage Example**: +```python +# Import and set up schema +# Refer to graphene_django.py or flask_ariadne.py for details. +``` + +--- + +### gRPC + +gRPC is a high-performance, RPC framework that uses protocol buffers to serialize data and is suitable for microservices communication. + +- **gRPC Python Client** (`grpc_python.py`): Python client for making gRPC calls. +- **Proto File** (`service.proto`): Defines the gRPC service and message structure. +- **Generated Code** (`service_pb2.py` and `service_pb2_grpc.py`): Generated from the proto file for server and client communication. + +**Usage Example**: +```python +# Run `python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service.proto` to generate service_pb2.py files. +# See grpc_python.py for client usage. +``` + +--- + +### MQTT + +MQTT is a lightweight messaging protocol often used in IoT applications for real-time communication. + +- **MQTT Client Example** (`mqtt.py`): Demonstrates setting up an MQTT client to publish and subscribe to topics using the `paho-mqtt` library. + +**Usage Example**: +```python +# Set up the MQTT client and connect to a broker. +# Refer to mqtt.py for detailed implementation. +``` + +--- + +### REST APIs + +REST APIs are widely used for web services, providing easy access to resources over HTTP. + +- **Django Rest Framework** (`django_rest.py`): Uses Django and DRF to set up a REST API. +- **Django Ninja** (`django_ninja_rest.py`): A fast alternative to DRF using Django Ninja with Python type hints. +- **Flask-RESTful** (`flask_rest.py`): REST API setup using Flask and Flask-RESTful. + +**Usage Example**: +```python +# Configure views and endpoints as per the specific framework. +# See django_rest.py, django_ninja_rest.py, and flask_rest.py for setup. +``` + +--- + +### SOAP + +SOAP is a protocol for web services that relies on XML and is often used in enterprise environments. + +- **Zeep with Python** (`zeep_python.py`): Demonstrates making SOAP API calls using the `zeep` library for a simple client interface. + +**Usage Example**: +```python +# Import zeep and configure a client using the WSDL URL. +# Refer to zeep_python.py for making calls to SOAP services. +``` + +--- + +### WebSockets + +WebSockets provide real-time, two-way communication, often used in chat applications, live notifications, and streaming. + +- **Django Channels** (`django_channels.py`): Uses Django Channels to handle WebSocket connections. +- **Flask-SocketIO** (`flask_sockets.py`): Implements WebSocket connections in Flask using Flask-SocketIO. + +**Usage Example**: +```python +# Set up WebSocket consumers or event handlers for real-time communication. +# See django_channels.py and flask_sockets.py for specific implementations. +``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `apis` folder. +2. Install any dependencies listed in each script. +3. Run the scripts to start the API server or client for each type. +4. Refer to each file for specific configuration and usage details. + +## Contributing + +Contributions are welcome! If you’d like to add more API types or enhance existing ones, please submit a pull request with a detailed description of your changes. diff --git a/apis/graphql/flask_ariadne.py b/apis/graphql/flask_ariadne.py new file mode 100644 index 0000000..194b57e --- /dev/null +++ b/apis/graphql/flask_ariadne.py @@ -0,0 +1,31 @@ +from ariadne import QueryType, make_executable_schema, graphql_sync +from flask import Flask, request, jsonify + +app = Flask(__name__) + +type_defs = """ + type Query { + hello: String! + } +""" + +query = QueryType() + + +@query.field("hello") +def resolve_hello(_, info): + return "Hello, world!" + + +schema = make_executable_schema(type_defs, query) + + +@app.route("/graphql", methods=["POST"]) +def graphql_server(): + data = request.get_json() + success, result = graphql_sync(schema, data) + return jsonify(result) + + +if __name__ == "__main__": + app.run() diff --git a/apis/graphql/graphene_django/main.py b/apis/graphql/graphene_django/main.py new file mode 100644 index 0000000..e46e793 --- /dev/null +++ b/apis/graphql/graphene_django/main.py @@ -0,0 +1,11 @@ +import graphene + + +class Query(graphene.ObjectType): + hello = graphene.String() + + def resolve_hello(self, info): + return "Hello from Graphene!" + + +schema = graphene.Schema(query=Query) diff --git a/apis/grpc/grpc_python.py b/apis/grpc/grpc_python.py new file mode 100644 index 0000000..76a298d --- /dev/null +++ b/apis/grpc/grpc_python.py @@ -0,0 +1,22 @@ +from concurrent import futures + +import grpc +import service_pb2 +import service_pb2_grpc + + +class Greeter(service_pb2_grpc.GreeterServicer): + def SayHello(self, request, context): + return service_pb2.HelloReply(message=f"Hello, {request.name}") + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + service_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + server.add_insecure_port('[::]:50051') + server.start() + server.wait_for_termination() + + +if __name__ == "__main__": + serve() diff --git a/apis/grpc/service.proto b/apis/grpc/service.proto new file mode 100644 index 0000000..4a3a771 --- /dev/null +++ b/apis/grpc/service.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +service Greeter { + rpc SayHello (HelloRequest) returns (HelloReply) {} +} + +message HelloRequest { + string name = 1; +} + +message HelloReply { + string message = 1; +} diff --git a/apis/grpc/service_pb2.py b/apis/grpc/service_pb2.py new file mode 100644 index 0000000..4d27c0c --- /dev/null +++ b/apis/grpc/service_pb2.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: service.proto +# Protobuf Python Version: 5.27.2 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 2, + '', + 'service.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rservice.proto\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t23\n\x07Greeter\x12(\n\x08SayHello\x12\r.HelloRequest\x1a\x0b.HelloReply\"\x00\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'service_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_HELLOREQUEST']._serialized_start=17 + _globals['_HELLOREQUEST']._serialized_end=45 + _globals['_HELLOREPLY']._serialized_start=47 + _globals['_HELLOREPLY']._serialized_end=76 + _globals['_GREETER']._serialized_start=78 + _globals['_GREETER']._serialized_end=129 +# @@protoc_insertion_point(module_scope) diff --git a/apis/grpc/service_pb2_grpc.py b/apis/grpc/service_pb2_grpc.py new file mode 100644 index 0000000..e077373 --- /dev/null +++ b/apis/grpc/service_pb2_grpc.py @@ -0,0 +1,97 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import service_pb2 as service__pb2 + +GRPC_GENERATED_VERSION = '1.67.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in service_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class GreeterStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.SayHello = channel.unary_unary( + '/Greeter/SayHello', + request_serializer=service__pb2.HelloRequest.SerializeToString, + response_deserializer=service__pb2.HelloReply.FromString, + _registered_method=True) + + +class GreeterServicer(object): + """Missing associated documentation comment in .proto file.""" + + def SayHello(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_GreeterServicer_to_server(servicer, server): + rpc_method_handlers = { + 'SayHello': grpc.unary_unary_rpc_method_handler( + servicer.SayHello, + request_deserializer=service__pb2.HelloRequest.FromString, + response_serializer=service__pb2.HelloReply.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'Greeter', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('Greeter', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class Greeter(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def SayHello(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/Greeter/SayHello', + service__pb2.HelloRequest.SerializeToString, + service__pb2.HelloReply.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/apis/mqtt/mqtt.py b/apis/mqtt/mqtt.py new file mode 100644 index 0000000..c5c86cb --- /dev/null +++ b/apis/mqtt/mqtt.py @@ -0,0 +1,25 @@ +import paho.mqtt.client as mqtt + +def on_connect(client, userdata, flags, rc): + print(f"Connected with result code {rc}") + client.subscribe("test/topic") + +def on_message(client, userdata, msg): + print(f"{msg.topic} {msg.payload.decode()}") + +client = mqtt.Client() + +client.on_connect = on_connect +client.on_message = on_message + +client.connect("mqtt.eclipseprojects.io", 1883, 60) + +client.loop_start() + +client.publish("test/topic", "Hello MQTT!") + +import time +time.sleep(10) + +client.loop_stop() +client.disconnect() diff --git a/apis/rest_apis/django_ninja_rest.py b/apis/rest_apis/django_ninja_rest.py new file mode 100644 index 0000000..9711c77 --- /dev/null +++ b/apis/rest_apis/django_ninja_rest.py @@ -0,0 +1,8 @@ +from ninja import NinjaAPI + +api = NinjaAPI() + + +@api.get("/hello/{name}") +def hello(request, name: str): + return {"message": f"Hello, {name}, from Django Ninja!"} diff --git a/apis/rest_apis/django_rest.py b/apis/rest_apis/django_rest.py new file mode 100644 index 0000000..9c16b34 --- /dev/null +++ b/apis/rest_apis/django_rest.py @@ -0,0 +1,17 @@ +# views.py +from rest_framework.decorators import api_view +from rest_framework.response import Response + + +@api_view(['GET']) +def api_root(request): + return Response({"message": "Welcome to Django REST API!"}) + + +# urls.py +from django.urls import path +from . import views + +urlpatterns = [ + path('', views.api_root, name='api_root'), +] diff --git a/apis/rest_apis/flask_rest.py b/apis/rest_apis/flask_rest.py new file mode 100644 index 0000000..66d8130 --- /dev/null +++ b/apis/rest_apis/flask_rest.py @@ -0,0 +1,16 @@ +from flask import Flask +from flask_restful import Resource, Api + +app = Flask(__name__) +api = Api(app) + + +class HelloWorld(Resource): + def get(self): + return {"message": "Hello, world!"} + + +api.add_resource(HelloWorld, '/') + +if __name__ == '__main__': + app.run(debug=True) diff --git a/apis/soap/zeep_python.py b/apis/soap/zeep_python.py new file mode 100644 index 0000000..891a05e --- /dev/null +++ b/apis/soap/zeep_python.py @@ -0,0 +1,28 @@ +import requests + +# Define the SOAP API endpoint +url = "http://www.dneonline.com/calculator.asmx" + +# Construct the SOAP request headers +headers = { + "Content-Type": "text/xml; charset=utf-8", + "SOAPAction": "http://tempuri.org/Add" +} + +# Constructing the SOAP XML body (for the 'Add' operation) +body = """ + + + + 5 + 10 + + +""" + +response = requests.post(url, data=body, headers=headers) + +print("Status Code:", response.status_code) +print("Response XML:", response.text) diff --git a/apis/web_sockets/django_channels.py b/apis/web_sockets/django_channels.py new file mode 100644 index 0000000..914d168 --- /dev/null +++ b/apis/web_sockets/django_channels.py @@ -0,0 +1,7 @@ +# consumers.py +from channels.generic.websocket import WebsocketConsumer + +class ChatConsumer(WebsocketConsumer): + def connect(self): + self.accept() + self.send(text_data="Welcome to WebSocket!") diff --git a/apis/web_sockets/flask_sockets.py b/apis/web_sockets/flask_sockets.py new file mode 100644 index 0000000..842843d --- /dev/null +++ b/apis/web_sockets/flask_sockets.py @@ -0,0 +1,12 @@ +from flask import Flask +from flask_socketio import SocketIO + +app = Flask(__name__) +socketio = SocketIO(app) + +@socketio.on('message') +def handle_message(msg): + print(f"Received message: {msg}") + +if __name__ == '__main__': + socketio.run(app) diff --git a/caching/README.md b/caching/README.md new file mode 100644 index 0000000..e6cd2e4 --- /dev/null +++ b/caching/README.md @@ -0,0 +1,174 @@ +# Caching + +This folder contains various caching implementations in Python, each suited for different use cases, from in-memory caching to file-based and distributed caching solutions. + +## Contents + +- [Redis Cache](#redis-cache) +- [Database Cache (SQLite)](#database-cache-sqlite) +- [Disk-Based Cache](#disk-based-cache) +- [FIFO Cache](#fifo-cache) +- [LIFO Cache](#lifo-cache) +- [LFU Cache](#lfu-cache) +- [LRU Cache](#lru-cache) +- [Memcached](#memcached) + +--- + +### Redis Cache + +**Redis Cache** is a high-performance distributed caching solution, often used for real-time applications. + +- **Files**: + - `main.py`: Example of connecting to Redis and using it for caching. + - `redis.conf`: Configuration file for Redis server. + +- **Setup**: + 1. Install the Redis Python client: `pip install redis`. + 2. Configure the `redis.conf` file as needed. + 3. Start Redis server and run `main.py`. + +- **Example Usage**: + ```python + from redis_cache.main import cache_set, cache_get + cache_set("key", "value") + print(cache_get("key")) + ``` + +--- + +### Database Cache (SQLite) + +**Database Cache** uses SQLite to store cache data persistently, making it useful for caching small datasets that need to survive restarts. + +- **File**: `db_cache.py` +- **Setup**: + 1. No additional setup required if Python's SQLite library is available. + +- **Example Usage**: + ```python + from db_cache import get_from_cache, set_in_cache + set_in_cache("user_id", "12345") + print(get_from_cache("user_id")) + ``` + +--- + +### Disk-Based Cache + +**Disk-Based Cache** uses file storage for caching, making it persistent across application restarts. + +- **File**: `disk_cache.py` +- **Setup**: + 1. Install `diskcache`: `pip install diskcache`. + +- **Example Usage**: + ```python + from disk_cache import disk_cache_set, disk_cache_get + disk_cache_set("username", "admin") + print(disk_cache_get("username")) + ``` + +--- + +### FIFO Cache + +**FIFO (First-In, First-Out) Cache** evicts the oldest cache entries first, making it ideal for cases where the order of data matters. + +- **File**: `fifo_cache.py` + +- **Example Usage**: + ```python + from fifo_cache import FIFOCache + cache = FIFOCache(3) + cache.put("a", 1) + cache.put("b", 2) + cache.put("c", 3) + cache.put("d", 4) # "a" is evicted + ``` + +--- + +### LIFO Cache + +**LIFO (Last-In, First-Out) Cache** evicts the most recently added item first, useful for stack-like data storage. + +- **File**: `lifo_cache.py` + +- **Example Usage**: + ```python + from lifo_cache import LIFOCache + cache = LIFOCache(3) + cache.put("x", 10) + cache.put("y", 20) + cache.put("z", 30) + cache.put("a", 40) # "z" is evicted + ``` + +--- + +### LFU Cache + +**LFU (Least Frequently Used) Cache** evicts the least accessed items, suitable for scenarios with infrequent access patterns. + +- **File**: `lfu_cache.py` + +- **Example Usage**: + ```python + from lfu_cache import LFUCache + cache = LFUCache(3) + cache.put("apple", 1) + cache.put("banana", 2) + cache.get("apple") + cache.put("cherry", 3) + cache.put("date", 4) # "banana" is evicted + ``` + +--- + +### LRU Cache + +**LRU (Least Recently Used) Cache** removes the least recently accessed items, making it suitable for applications where data accessed frequently is retained. + +- **File**: `lru_cache.py` + +- **Example Usage**: + ```python + from lru_cache import LRUCache + cache = LRUCache(3) + cache.put("key1", "value1") + cache.put("key2", "value2") + cache.get("key1") + cache.put("key3", "value3") + cache.put("key4", "value4") # "key2" is evicted + ``` + +--- + +### Memcached + +**Memcached** is an in-memory caching solution designed for distributed systems. + +- **File**: `memcached.py` +- **Setup**: + 1. Install `python-memcached`: `pip install python-memcached`. + 2. Start a Memcached server. + +- **Example Usage**: + ```python + from memcached import memcached_set, memcached_get + memcached_set("session_id", "ABC123") + print(memcached_get("session_id")) + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `caching` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure and run each script as needed for the caching strategy you want to implement. + +## Contributing + +Contributions are welcome! If you’d like to add more caching techniques or improve existing ones, please submit a pull request with a detailed description of your changes. diff --git a/caching/db_cache.py b/caching/db_cache.py new file mode 100644 index 0000000..ef58bd9 --- /dev/null +++ b/caching/db_cache.py @@ -0,0 +1,32 @@ +import sqlite3 + +# Connect to SQLite database (or create it if it doesn't exist) +conn = sqlite3.connect('cache.db') +cursor = conn.cursor() + +# Create a cache table +cursor.execute(''' + CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + value TEXT, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP + ) +''') +conn.commit() + + +# Function to get cached value +def get_from_cache(key): + cursor.execute("SELECT value FROM cache WHERE key = ?", (key,)) + result = cursor.fetchone() + return result[0] if result else None + + +def set_in_cache(key, value): + cursor.execute("INSERT OR REPLACE INTO cache (key, value) VALUES (?, ?)", (key, value)) + conn.commit() + + +# Example usage +set_in_cache("username", "JohnDoe") +print(get_from_cache("username")) # If all is well it should return "JohnDoe" diff --git a/caching/disk_cache.py b/caching/disk_cache.py new file mode 100644 index 0000000..3b28bb1 --- /dev/null +++ b/caching/disk_cache.py @@ -0,0 +1,15 @@ +import diskcache as dc + +cache = dc.Cache('/tmp/my_cache') + +def expensive_computation(n): + if n in cache: + return cache[n] + else: + result = n * n + cache[n] = result + return result + +# Example usage +print(expensive_computation(5)) # Computed and saved to disk cache +print(expensive_computation(5)) # Retrieved from disk cache diff --git a/caching/fifo_cache.py b/caching/fifo_cache.py new file mode 100644 index 0000000..1fc5b55 --- /dev/null +++ b/caching/fifo_cache.py @@ -0,0 +1,27 @@ +from collections import OrderedDict + +class FIFOCache: + def __init__(self, capacity): + self.cache = OrderedDict() + self.capacity = capacity + + def get(self, key): + return self.cache.get(key, -1) + + def put(self, key, value): + if key in self.cache: + self.cache.move_to_end(key) + self.cache[key] = value + if len(self.cache) > self.capacity: + self.cache.popitem(last=False) # Remove the first item (FIFO) + +# Example usage +cache = FIFOCache(3) +cache.put("a", 1) +cache.put("b", 2) +cache.put("c", 3) +print(cache.cache) + +# Add a new item +cache.put("d", 4) +print(cache.cache) # "a" should be evicted since it's the oldest diff --git a/caching/lfu_cache.py b/caching/lfu_cache.py new file mode 100644 index 0000000..d53013b --- /dev/null +++ b/caching/lfu_cache.py @@ -0,0 +1,19 @@ +from cachetools import LFUCache + +# Define an LFU cache with a max size of 3 items +cache = LFUCache(maxsize=3) + +# Insert items into the cache +cache['a'] = 1 +cache['b'] = 2 +cache['c'] = 3 + +# Access items to change their frequency +print(cache['a']) # Access 'a' +print(cache['b']) # Access 'b' + +# Add a new item, evicting the least frequently used (LFU) +cache['d'] = 4 + +# 'c' should be evicted since it's the least frequently accessed +print(cache) diff --git a/caching/lifo_cache.py b/caching/lifo_cache.py new file mode 100644 index 0000000..f942f17 --- /dev/null +++ b/caching/lifo_cache.py @@ -0,0 +1,27 @@ +from collections import OrderedDict + + +class LIFOCache: + def __init__(self, capacity): + self.cache = OrderedDict() + self.capacity = capacity + + def get(self, key): + return self.cache.get(key, -1) + + def put(self, key, value): + self.cache[key] = value + if len(self.cache) > self.capacity: + self.cache.popitem(last=True) # Remove the last item (LIFO) + + +# Example usage +cache = LIFOCache(3) +cache.put("a", 1) +cache.put("b", 2) +cache.put("c", 3) +print(cache.cache) + +# Add a new item +cache.put("d", 4) +print(cache.cache) # "c" should be evicted since it's the most recent diff --git a/caching/lru_cache.py b/caching/lru_cache.py new file mode 100644 index 0000000..a1db96a --- /dev/null +++ b/caching/lru_cache.py @@ -0,0 +1,10 @@ +from functools import lru_cache + +@lru_cache(maxsize=128) +def expensive_computation(n): + print(f"Computing {n}...") + return n * n + +# Example usage +print(expensive_computation(5)) # Computed and cached +print(expensive_computation(5)) # Retrieved from cache diff --git a/caching/memcached.py b/caching/memcached.py new file mode 100644 index 0000000..4329886 --- /dev/null +++ b/caching/memcached.py @@ -0,0 +1,18 @@ +import memcache + +# Connect to Memcached +cache = memcache.Client([('127.0.0.1', 11211)]) + +def expensive_computation(n): + # Check if result is cached + result = cache.get(str(n)) + if result: + return result + else: + result = n * n + cache.set(str(n), result, time=60) # time in seconds + return result + +# Example usage +print(expensive_computation(5)) # Computed and saved to Memcached +print(expensive_computation(5)) # Retrieved from Memcached cache diff --git a/caching/redis_cache/main.py b/caching/redis_cache/main.py new file mode 100644 index 0000000..a241ec4 --- /dev/null +++ b/caching/redis_cache/main.py @@ -0,0 +1,28 @@ +import redis +import os +from dotenv import load_dotenv + +load_dotenv() + +REDIS_HOST = os.getenv("REDIS_HOST") +REDIS_PORT = os.getenv("REDIS_PORT") +REDIS_PASSWORD = os.getenv("REDIS_PASSWORD") + +cache = redis.StrictRedis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PASSWORD, + decode_responses=True, +) + +def expensive_computation(n): + if cache.exists(n): + return cache.get(n) + else: + result = n * n + cache.set(n, result, ex=60) + return result + +# Example usage +print(expensive_computation(5)) # Computed and saved to Redis +print(expensive_computation(5)) # Retrieved from Redis cache diff --git a/caching/redis_cache/redis.conf b/caching/redis_cache/redis.conf new file mode 100644 index 0000000..3716fa7 --- /dev/null +++ b/caching/redis_cache/redis.conf @@ -0,0 +1 @@ +requirepass your_redis_password diff --git a/clean_architecture/README.md b/clean_architecture/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/clean_architecture/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/database/README.md b/database/README.md new file mode 100644 index 0000000..e78555e --- /dev/null +++ b/database/README.md @@ -0,0 +1,192 @@ +# Databases + +This folder contains Python scripts for connecting to and interacting with various types of databases, each suited for different use cases and data structures. Each script provides functions for common database operations such as connecting, querying, inserting, updating, and deleting records. + +## Contents + +- [MySQL Database](#mysql-database) +- [PostgreSQL Database](#postgresql-database) +- [SQLite Database](#sqlite-database) +- [MongoDB Database](#mongodb-database) +- [Redis Database](#redis-database) +- [Cassandra Database](#cassandra-database) +- [Neo4j Database](#neo4j-database) +- [Elasticsearch](#elasticsearch) +- [Amazon Aurora](#amazon-aurora) + +--- + +### MySQL Database + +**MySQL** is a widely used open-source relational database that’s known for its reliability and ease of use. + +- **File**: `mysql_database.py` +- **Setup**: + 1. Install MySQL client library: `pip install mysql-connector-python`. + 2. Configure MySQL credentials. + +- **Example Usage**: + ```python + from mysql_database import connect_to_mysql, execute_query + connection = connect_to_mysql() + execute_query(connection, "SELECT * FROM my_table") + ``` + +--- + +### PostgreSQL Database + +**PostgreSQL** is a powerful, open-source relational database system known for its robustness and support for complex queries. + +- **File**: `postgresql_database.py` +- **Setup**: + 1. Install PostgreSQL client library: `pip install psycopg2`. + 2. Configure PostgreSQL credentials. + +- **Example Usage**: + ```python + from postgresql_database import connect_to_postgresql, execute_query + connection = connect_to_postgresql() + execute_query(connection, "SELECT * FROM my_table") + ``` + +--- + +### SQLite Database + +**SQLite** is a lightweight, file-based relational database often used for small-scale applications and testing. + +- **File**: `sqlite_database.py` +- **Setup**: + - No additional installation is required as SQLite is included with Python. + +- **Example Usage**: + ```python + from sqlite_database import connect_to_sqlite, execute_query + connection = connect_to_sqlite("database.db") + execute_query(connection, "SELECT * FROM my_table") + ``` + +--- + +### MongoDB Database + +**MongoDB** is a NoSQL database known for its flexibility and scalability, making it suitable for large datasets and unstructured data. + +- **File**: `mongodb_database.py` +- **Setup**: + 1. Install MongoDB client library: `pip install pymongo`. + 2. Configure MongoDB connection details. + +- **Example Usage**: + ```python + from mongodb_database import connect_to_mongodb, insert_document + db = connect_to_mongodb("my_database") + insert_document(db, "my_collection", {"name": "Alice", "age": 30}) + ``` + +--- + +### Redis Database + +**Redis** is an in-memory key-value store, often used for caching and real-time analytics. + +- **File**: `redis_database.py` +- **Setup**: + 1. Install Redis client library: `pip install redis`. + 2. Configure Redis connection settings. + +- **Example Usage**: + ```python + from redis_database import connect_to_redis, set_key, get_key + redis_client = connect_to_redis() + set_key(redis_client, "username", "john_doe") + print(get_key(redis_client, "username")) + ``` + +--- + +### Cassandra Database + +**Cassandra** is a distributed NoSQL database designed to handle large amounts of data across many servers, providing high availability. + +- **File**: `cassandra_database.py` +- **Setup**: + 1. Install Cassandra driver: `pip install cassandra-driver`. + 2. Configure Cassandra cluster connection. + +- **Example Usage**: + ```python + from cassandra_database import connect_to_cassandra, execute_query + session = connect_to_cassandra() + execute_query(session, "SELECT * FROM my_table") + ``` + +--- + +### Neo4j Database + +**Neo4j** is a graph database that stores data in nodes and relationships, making it suitable for applications involving complex relationships. + +- **File**: `neo4j_database.py` +- **Setup**: + 1. Install Neo4j client library: `pip install neo4j`. + 2. Configure Neo4j connection settings. + +- **Example Usage**: + ```python + from neo4j_database import connect_to_neo4j, execute_cypher_query + driver = connect_to_neo4j() + execute_cypher_query(driver, "MATCH (n) RETURN n") + ``` + +--- + +### Elasticsearch + +**Elasticsearch** is a distributed search and analytics engine often used for log and text data storage and querying. + +- **File**: `elasticsearch_database.py` +- **Setup**: + 1. Install Elasticsearch client: `pip install elasticsearch`. + 2. Configure Elasticsearch connection settings. + +- **Example Usage**: + ```python + from elasticsearch_database import connect_to_elasticsearch, search_index + es = connect_to_elasticsearch() + search_results = search_index(es, "my_index", {"query": {"match_all": {}}}) + ``` + +--- + +### Amazon Aurora + +**Amazon Aurora** is a MySQL and PostgreSQL-compatible relational database built for the cloud, providing high performance and availability. + +- **File**: `aurora_database.py` +- **Setup**: + 1. Install the MySQL or PostgreSQL client library (depending on Aurora's compatibility mode): + - For MySQL: `pip install mysql-connector-python` + - For PostgreSQL: `pip install psycopg2` + 2. Configure Aurora database endpoint, username, password, and database name. + +- **Example Usage**: + ```python + from aurora_database import connect_to_aurora, execute_query + connection = connect_to_aurora() + execute_query(connection, "SELECT * FROM my_table") + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `databases` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your database credentials or connection settings for each database type. +4. Use the example code snippets in each script to connect to, query, or manipulate data in the respective database. + +## Contributing + +Contributions are welcome! If you’d like to add more database types or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/deployments/README.md b/deployments/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/deployments/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/docker/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/emails/README.md b/emails/README.md new file mode 100644 index 0000000..93929e1 --- /dev/null +++ b/emails/README.md @@ -0,0 +1,127 @@ +# Emails + +This folder contains Python scripts for integrating various email service providers and sending emails using different clients. Each script demonstrates how to set up and send emails, with optional attachment support, using popular email APIs and protocols. + +## Contents + +- [AWS SES](#aws-ses) +- [Azure Email](#azure-email) +- [Mailersend](#mailersend) +- [Mailgun](#mailgun) +- [SendGrid](#sendgrid) +- [SMTP](#smtp) + +--- + +### AWS SES + +**AWS Simple Email Service (SES)** is a cloud-based email sending service from Amazon, suitable for transactional and marketing emails. + +- **File**: `aws_ses_email.py` +- **Setup**: + 1. Install the AWS SDK: `pip install boto3`. + 2. Configure AWS credentials with access to SES. + +- **Example Usage**: + ```python + from aws_ses_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +### Azure Email + +**Azure Email** is part of Azure Communication Services, providing an API for sending transactional and notification emails. + +- **File**: `azure_email.py` +- **Setup**: + 1. Install Azure SDK: `pip install azure-communication-email`. + 2. Configure Azure credentials (Azure Communication Services connection string). + +- **Example Usage**: + ```python + from azure_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +### Mailersend + +**Mailersend** is a transactional email service designed for developers to integrate email sending through a simple API. + +- **File**: `mailersend_email.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Get the Mailersend API key. + +- **Example Usage**: + ```python + from mailersend_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +### Mailgun + +**Mailgun** is a popular email API for developers, especially for handling transactional emails and email validation. + +- **File**: `mailgun_email.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Get the Mailgun API key and domain. + +- **Example Usage**: + ```python + from mailgun_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +### SendGrid + +**SendGrid** is a cloud-based email delivery service for transactional and marketing emails, providing analytics and deliverability features. + +- **File**: `sendgrid_email.py` +- **Setup**: + 1. Install the SendGrid SDK: `pip install sendgrid`. + 2. Get the SendGrid API key. + +- **Example Usage**: + ```python + from sendgrid_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +### SMTP + +**SMTP (Simple Mail Transfer Protocol)** is the standard protocol for sending emails. This script demonstrates sending emails using Python’s `smtplib`, suitable for any SMTP server. + +- **File**: `smtp_email.py` +- **Setup**: + 1. No additional packages required; uses built-in `smtplib`. + 2. Configure SMTP server settings (e.g., Gmail, Outlook). + +- **Example Usage**: + ```python + from smtp_email import send_email + send_email("recipient@example.com", "Subject", "Message body", "/path/to/attachment.txt") + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `emails` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your credentials for each service (e.g., API keys, SMTP server details). +4. Run the scripts to send emails using the specified email client. + +## Contributing + +Contributions are welcome! If you’d like to add more email providers or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/emails/aws_ses_email.py b/emails/aws_ses_email.py new file mode 100644 index 0000000..30683d7 --- /dev/null +++ b/emails/aws_ses_email.py @@ -0,0 +1,65 @@ +import os +from email import encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +import boto3 + +AWS_REGION = os.getenv('AWS_REGION') +AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') + + +def send_aws_ses_email(to_email, subject, message_body, file_path=None): + """ + Sends an email with an attachment using AWS SES. + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): Optional. Path to the file to be attached. + + Returns: + dict: Response from AWS SES. + """ + client = boto3.client( + 'ses', + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY + ) + + msg = MIMEMultipart() + msg['From'] = 'a_verified_email@example.com' + msg['To'] = to_email + msg['Subject'] = subject + msg.attach(MIMEText(message_body, 'html')) + + with open(file_path, 'rb') as attachment: + part = MIMEBase('application', 'octet-stream') + part.set_payload(attachment.read()) + encoders.encode_base64(part) + part.add_header('Content-Disposition', f'attachment; filename= {os.path.basename(file_path)}') + msg.attach(part) + + try: + response = client.send_raw_email( + Source=msg['From'], + Destinations=[to_email], + RawMessage={'Data': msg.as_string()} + ) + print(f"Email sent successfully! Message ID: {response['MessageId']}") + except Exception as e: + print(f"Error sending email: {str(e)}") + + +# Example usage +if __name__ == "__main__": + send_aws_ses_email( + 'recipient@example.com', + 'Test Email with Attachment from SES', + 'Hello from AWS SES!', + '/tmp/my-file.xlsx', + ) diff --git a/emails/azure_email.py b/emails/azure_email.py new file mode 100644 index 0000000..b10280a --- /dev/null +++ b/emails/azure_email.py @@ -0,0 +1,57 @@ +import base64 +import os + +from azure.communication.email import EmailClient +from dotenv import load_dotenv + +load_dotenv() + +AZURE_COMMUNICATION_CONNECTION_STRING = os.getenv('AZURE_COMMUNICATION_CONNECTION_STRING') + + +def send_azure_email(to_email, subject, message_body, file_path=None): + """ + Sends an email with an attachment using Azure Communication Services. + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): The file path to the attachment. + + Returns: + dict: Response from Azure Communication Services. + """ + client = EmailClient.from_connection_string(AZURE_COMMUNICATION_CONNECTION_STRING) + + with open(file_path, 'rb') as f: + file_content = base64.b64encode(f.read()).decode('utf-8') + + message = { + "content": { + "subject": subject, + "html": message_body + }, + "recipients": { + "to": [{"address": to_email, "displayName": "Recipient"}] + }, + "senderAddress": "your_verified_email@example.com", + "attachments": [ + { + "name": os.path.basename(file_path), + "attachmentType": "application/octet-stream", + "contentInBase64": file_content + } + ] + } + + poller = client.begin_send(message) + + result = poller.result() + print(f"Email sent successfully with operation ID: {result['id']}") + + +# Example usage +if __name__ == "__main__": + send_azure_email('recipient@example.com', 'Test Email with Attachment from Azure', + 'Hello from Azure!', '/tmp/my-file.xlsx') diff --git a/emails/mailersend_email.py b/emails/mailersend_email.py new file mode 100644 index 0000000..0845d4e --- /dev/null +++ b/emails/mailersend_email.py @@ -0,0 +1,60 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +MAILERSEND_API_KEY = os.getenv('MAILERSEND_API_KEY') + + +def send_mailersend_email(to_email, subject, message_body, file_path): + """ + Sends an email with an attachment using MailerSend. + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): The file path to the attachment. + + Returns: + dict: Response from MailerSend API. + """ + with open(file_path, 'rb') as f: + files = { + 'attachments': (os.path.basename(file_path), f, 'application/octet-stream') + } + + data = { + "from": { + "email": "your_verified_email@example.com", + "name": "Your Name" + }, + "to": [ + {"email": to_email, "name": "Recipient Name"} + ], + "subject": subject, + "html": message_body + } + + headers = { + "Authorization": f"Bearer {MAILERSEND_API_KEY}", + "Content-Type": "application/json" + } + + email_response = requests.post( + "https://api.mailersend.com/v1/email", + headers=headers, + json=data, + files=files + ) + + return email_response.json() + + +# Example usage +if __name__ == "__main__": + response = send_mailersend_email('recipient@example.com', 'Test Email with Attachment from MailerSend', + 'Hello from MailerSend!', '/tmp/my-file.xlsx') + print(response) diff --git a/emails/mailgun_email.py b/emails/mailgun_email.py new file mode 100644 index 0000000..e1cbbeb --- /dev/null +++ b/emails/mailgun_email.py @@ -0,0 +1,43 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +MAILGUN_API_KEY = os.getenv('MAILGUN_API_KEY') +MAILGUN_DOMAIN = os.getenv('MAILGUN_DOMAIN') + + +def send_mailgun_email(to_email, subject, message_body, file_path=None): + """ + Sends an email with an attachment using Mailgun. + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): Optional. The file path to the attachment. + + Returns: + dict: Response from Mailgun API. + """ + with open(file_path, 'rb') as attachment: + return requests.post( + f"https://api.mailgun.net/v3/{MAILGUN_DOMAIN}/messages", + auth=("api", MAILGUN_API_KEY), + files=[("attachment", attachment)], + data={ + "from": "Your Name ", + "to": to_email, + "subject": subject, + "html": message_body + } + ) + + +# Example usage +if __name__ == "__main__": + response = send_mailgun_email('recipient@example.com', 'Test Email with Attachment from Mailgun', + 'Hello from Mailgun!', '/tmp/my-file.xlsx') + print(response.text) diff --git a/emails/sendgrid_email.py b/emails/sendgrid_email.py new file mode 100644 index 0000000..0c978a7 --- /dev/null +++ b/emails/sendgrid_email.py @@ -0,0 +1,61 @@ +import base64 +import os + +from dotenv import load_dotenv +from sendgrid import SendGridAPIClient +from sendgrid.helpers.mail import Mail, Attachment, FileContent, FileName, FileType, Disposition + +load_dotenv() + +SENDGRID_API_KEY = os.getenv('SENDGRID_API_KEY') + + +def send_sendgrid_email(to_email, subject, message_body, file_path=None): + """ + Sends an email using Twilio SendGrid, with optional file attachment. + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): Optional path to a file to send as an attachment. + + Returns: + dict: Response from SendGrid. + """ + message = Mail( + from_email='your_verified_email@example.com', + to_emails=to_email, + subject=subject, + html_content=message_body + ) + + if file_path: + with open(file_path, 'rb') as f: + file_data = f.read() + encoded_file = base64.b64encode(file_data).decode('utf-8') + + attachment = Attachment( + FileContent(encoded_file), + FileName(os.path.basename(file_path)), + FileType('application/octet-stream'), + Disposition('attachment') + ) + message.attachment = attachment + + try: + sg = SendGridAPIClient(SENDGRID_API_KEY) + response = sg.send(message) + print(f"Email sent successfully with status code: {response.status_code}") + except Exception as e: + print(f"Error sending email: {str(e)}") + + +# Example usage +if __name__ == "__main__": + send_sendgrid_email( + 'recipient@example.com', + 'Test Email with Attachment', + 'Hello from SendGrid!', + '/path/to/file.xlsx', + ) diff --git a/emails/smtp_email.py b/emails/smtp_email.py new file mode 100644 index 0000000..e979679 --- /dev/null +++ b/emails/smtp_email.py @@ -0,0 +1,53 @@ +import os +import smtplib +from email import encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +SMTP_SERVER = "smtp.gmail.com" # This can be changed to any other SMTP server Office 365, yahoo, etc.) +SMTP_PORT = 587 # This can be changed to the appropriate port for the SMTP server +SMTP_USER = os.getenv('SMTP_USER') +SMTP_PASSWORD = os.getenv('SMTP_PASSWORD') + + +def send_smtp_email(to_email, subject, message_body, file_path=None): + """ + Sends an email with an attachment using an SMTP server (e.g., Gmail). + + Args: + to_email (str): Recipient's email address. + subject (str): Email subject. + message_body (str): The email message content. + file_path (str): The file path to the attachment. + + Returns: + None + """ + msg = MIMEMultipart() + msg['From'] = SMTP_USER + msg['To'] = to_email + msg['Subject'] = subject + msg.attach(MIMEText(message_body, 'html')) + + with open(file_path, 'rb') as attachment: + part = MIMEBase('application', 'octet-stream') + part.set_payload(attachment.read()) + encoders.encode_base64(part) + part.add_header('Content-Disposition', f'attachment; filename= {os.path.basename(file_path)}') + msg.attach(part) + + with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as server: + server.starttls() + server.login(SMTP_USER, SMTP_PASSWORD) + server.sendmail(SMTP_USER, to_email, msg.as_string()) + + +# Example usage +if __name__ == "__main__": + send_smtp_email( + 'recipient@example.com', + 'Test Email with Attachment from SMTP', + 'Hello from SMTP!', + '/tmp/my-file.xlsx', + ) diff --git a/env_secrets/README.md b/env_secrets/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/env_secrets/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/logging/README.md b/logging/README.md new file mode 100644 index 0000000..a996f36 --- /dev/null +++ b/logging/README.md @@ -0,0 +1,109 @@ +# Logging + +This folder contains Python implementations for various logging solutions, including integrations with popular cloud logging services and structured logging setups. Each script provides methods for logging application events, errors, and structured data. + +## Contents + +- [AWS CloudWatch Logging](#aws-cloudwatch-logging) +- [Azure Monitor Logging](#azure-monitor-logging) +- [Basic Logging](#basic-logging) +- [Google Cloud Logging](#google-cloud-logging) +- [Structured Logging](#structured-logging) + +--- + +### AWS CloudWatch Logging + +**AWS CloudWatch Logging** allows you to monitor, store, and access log files from Amazon services, including custom application logs. + +- **File**: `aws_cloudwatch_logging.py` +- **Setup**: + 1. Install AWS SDK: `pip install boto3`. + 2. Configure AWS credentials with access to CloudWatch Logs. + +- **Example Usage**: + ```python + from aws_cloudwatch_logging import setup_cloudwatch_logger + logger = setup_cloudwatch_logger() + logger.info("This is an info message sent to AWS CloudWatch.") + ``` + +--- + +### Azure Monitor Logging + +**Azure Monitor Logging** enables you to collect and analyze logs from applications and services on Azure. This is part of the broader Azure Monitor suite. + +- **File**: `azure_monitor_logging.py` +- **Setup**: + 1. Install Azure SDK: `pip install azure-monitor-opentelemetry`. + 2. Set up Azure Monitor with appropriate credentials. + +- **Example Usage**: + ```python + from azure_monitor_logging import setup_azure_logger + logger = setup_azure_logger() + logger.warning("This is a warning message sent to Azure Monitor.") + ``` + +--- + +### Basic Logging + +**Basic Logging** demonstrates how to set up logging in Python using the built-in `logging` module. This is useful for local development or simple logging needs. + +- **File**: `basic_logging.py` + +- **Example Usage**: + ```python + from basic_logging import setup_basic_logger + logger = setup_basic_logger() + logger.error("This is an error message in the basic logging setup.") + ``` + +--- + +### Google Cloud Logging + +**Google Cloud Logging** (formerly Stackdriver) provides a powerful logging service that integrates with other Google Cloud services. + +- **File**: `gcp_logging.py` +- **Setup**: + 1. Install Google Cloud logging client: `pip install google-cloud-logging`. + 2. Configure Google Cloud credentials for authentication. + +- **Example Usage**: + ```python + from gcp_logging import setup_gcp_logger + logger = setup_gcp_logger() + logger.debug("This is a debug message sent to Google Cloud Logging.") + ``` + +--- + +### Structured Logging + +**Structured Logging** involves logging data in a structured format (e.g., JSON), making it easier to query and analyze in log management tools. + +- **File**: `structured_logging.py` +- **Setup**: No additional dependencies required. + +- **Example Usage**: + ```python + from structured_logging import setup_structured_logger + logger = setup_structured_logger() + logger.info("This is a structured log message.", extra={"user": "admin", "action": "login"}) + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `logging` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure credentials for cloud logging services (AWS, Azure, GCP) where applicable. +4. Run the scripts to log messages to the respective logging service. + +## Contributing + +Contributions are welcome! If you’d like to add more logging solutions or improve existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/logging/aws_cloudwatch_logging.py b/logging/aws_cloudwatch_logging.py new file mode 100644 index 0000000..d2bf7ac --- /dev/null +++ b/logging/aws_cloudwatch_logging.py @@ -0,0 +1,21 @@ +import logging +import watchtower +from dotenv import load_dotenv +import os + +load_dotenv() + +AWS_REGION = os.getenv('AWS_REGION') +AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') + +logger = logging.getLogger('cloudwatchLogger') +logger.setLevel(logging.INFO) +handler = watchtower.CloudWatchLogHandler( + log_group='your-log-group', + stream_name='your-log-stream', + region_name=AWS_REGION +) +logger.addHandler(handler) + +logger.info('This is an info message sent to AWS CloudWatch') diff --git a/logging/azure_monitor_logging.py b/logging/azure_monitor_logging.py new file mode 100644 index 0000000..df25ed0 --- /dev/null +++ b/logging/azure_monitor_logging.py @@ -0,0 +1,27 @@ +import os +from datetime import datetime + +from azure.core.exceptions import HttpResponseError +from azure.identity import DefaultAzureCredential +from azure.monitor.ingestion import LogsIngestionClient +from dotenv import load_dotenv + +load_dotenv() + +endpoint = os.getenv('DATA_COLLECTION_ENDPOINT') +rule_id = os.getenv('LOGS_DCR_RULE_ID') +stream_name = os.getenv('LOGS_DCR_STREAM_NAME') + +credential = DefaultAzureCredential() +client = LogsIngestionClient(endpoint=endpoint, credential=credential, logging_enable=True) + +logs = [ + {"Time": datetime.utcnow().isoformat(), "Level": "INFO", "Message": "Log message sent to Azure Monitor"}, + {"Time": datetime.utcnow().isoformat(), "Level": "ERROR", "Message": "An error log message"} +] + +try: + client.upload(rule_id=rule_id, stream_name=stream_name, logs=logs) + print("Logs uploaded successfully") +except HttpResponseError as e: + print(f"Failed to upload logs: {e}") diff --git a/logging/basic_logging.py b/logging/basic_logging.py new file mode 100644 index 0000000..23eab4a --- /dev/null +++ b/logging/basic_logging.py @@ -0,0 +1,13 @@ +import logging + +logging.basicConfig( + filename='app.log', # File we want to log to + level=logging.DEBUG, # Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) + format='%(asctime)s - %(levelname)s - %(message)s', # Log format +) + +logging.debug('This is a debug message') +logging.info('This is an info message') +logging.warning('This is a warning message') +logging.error('This is an error message') +logging.critical('This is a critical message') diff --git a/logging/gcp_logging.py b/logging/gcp_logging.py new file mode 100644 index 0000000..89d3193 --- /dev/null +++ b/logging/gcp_logging.py @@ -0,0 +1,53 @@ +import logging +import os + +import google.cloud.logging +from dotenv import load_dotenv +from google.cloud.logging.handlers import CloudLoggingHandler + +load_dotenv() + +gcp_credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + +if not gcp_credentials_path: + raise EnvironmentError("GOOGLE_APPLICATION_CREDENTIALS environment variable not set") + +try: + client = google.cloud.logging.Client() +except Exception as e: + logging.error(f"Failed to initialize Google Cloud client: {e}") + raise + +try: + handler = CloudLoggingHandler(client) + logger = logging.getLogger('gcpLogger') + logger.setLevel(logging.INFO) + logger.addHandler(handler) +except Exception as e: + logging.error(f"Failed to set up Google Cloud Logging handler: {e}") + raise + + +def log_message(message, level=logging.INFO): + try: + if level == logging.INFO: + logger.info(message) + elif level == logging.WARNING: + logger.warning(message) + elif level == logging.ERROR: + logger.error(message) + elif level == logging.DEBUG: + logger.debug(message) + else: + logger.log(level, message) + print(f"Log sent to Google Cloud: {message}") + except Exception as e: + logging.error(f"Failed to send log to Google Cloud Logging: {e}") + print(f"Error sending log: {e}") + + +# Example usage +if __name__ == "__main__": + log_message("This is an info message sent to Google Cloud Logging", logging.INFO) + log_message("This is a warning message", logging.WARNING) + log_message("This is an error message", logging.ERROR) diff --git a/logging/structured_logging.py b/logging/structured_logging.py new file mode 100644 index 0000000..510f18c --- /dev/null +++ b/logging/structured_logging.py @@ -0,0 +1,24 @@ +import json +import logging + + +class JSONFormatter(logging.Formatter): + def format(self, record): + log_message = { + 'level': record.levelname, + 'message': record.getMessage(), + 'timestamp': self.formatTime(record, self.datefmt), + 'name': record.name + } + return json.dumps(log_message) + + +logger = logging.getLogger('structuredLogger') +handler = logging.FileHandler('structured_app.log') +formatter = JSONFormatter() +handler.setFormatter(formatter) +logger.addHandler(handler) +logger.setLevel(logging.DEBUG) + +logger.info('Structured info message') +logger.error('Structured error message') diff --git a/machine_learning/README.md b/machine_learning/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/machine_learning/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..78547d2 --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,130 @@ +# Monitoring + +This folder contains scripts and configurations for integrating various monitoring tools to track system performance, application health, and infrastructure metrics. Each directory includes specific instructions for setting up and using these tools, along with sample configurations. + +## Contents + +- [Checkmk](#checkmk) +- [Datadog](#datadog) +- [Elasticsearch](#elasticsearch) +- [Grafana](#grafana) +- [InfluxDB](#influxdb) +- [Prometheus](#prometheus) +- [Sentry](#sentry) +- [Zabbix](#zabbix) + +--- + +### Checkmk + +**Checkmk** is a comprehensive monitoring system that supports infrastructure, network, and application monitoring. + +- **Folder**: `checkmk` +- **Setup**: + 1. Deploy Checkmk server and configure agents on target hosts. + 2. Define checks and configure notifications as needed. + +- **Example Configuration**: Refer to the `checkmk` folder for sample configuration files. + +--- + +### Datadog + +**Datadog** provides cloud-based monitoring and analytics for infrastructure, applications, logs, and more. + +- **Folder**: `datadog` +- **Setup**: + 1. Install the Datadog Agent: `pip install datadog`. + 2. Configure the API key and set up metrics collection. + +- **Example Configuration**: Check the `datadog` folder for integration examples and sample code. + +--- + +### Elasticsearch + +**Elasticsearch** is a distributed search and analytics engine, commonly used in the ELK stack for logging and monitoring. + +- **Folder**: `elasticsearch` +- **Setup**: + 1. Install and configure Elasticsearch with Logstash and Kibana. + 2. Define index patterns and create visualizations in Kibana. + +- **Example Configuration**: Refer to the `elasticsearch` folder for Logstash configuration and setup instructions. + +--- + +### Grafana + +**Grafana** is an open-source visualization tool that integrates with various data sources like Prometheus, InfluxDB, and Elasticsearch. + +- **Folder**: `grafana` +- **Setup**: + 1. Configure Grafana and add data sources. + 2. Create custom dashboards to visualize metrics. + +- **Example Configuration**: Sample dashboard JSON files and data source configurations are available in the `grafana` folder. + +--- + +### InfluxDB + +**InfluxDB** is a time-series database optimized for storing metrics and events, often used with Grafana for visualizations. + +- **Folder**: `influxdb` +- **Setup**: + 1. Install and configure InfluxDB for metrics storage. + 2. Use Telegraf or custom scripts to send metrics to InfluxDB. + +- **Example Configuration**: Sample InfluxDB and Telegraf configurations can be found in the `influxdb` folder. + +--- + +### Prometheus + +**Prometheus** is a monitoring system and time-series database, primarily used for scraping metrics from applications and services. + +- **Folder**: `prometheus` +- **Setup**: + 1. Install Prometheus and configure scrape jobs for applications. + 2. Set up Alertmanager for notifications. + +- **Example Configuration**: Prometheus configuration files are available in the `prometheus` folder. + +--- + +### Sentry + +**Sentry** is an error-tracking and performance monitoring tool that captures exceptions and issues in real-time. + +- **Folder**: `sentry` +- **Setup**: + 1. Install the Sentry SDK in your application. + 2. Configure DSN and initialize Sentry for error monitoring. + +- **Example Configuration**: Refer to the `sentry` folder for setup and usage examples. + +--- + +### Zabbix + +**Zabbix** is an enterprise-level monitoring platform for networks, servers, and applications. + +- **Folder**: `zabbix` +- **Setup**: + 1. Install the Zabbix server and configure agents on target systems. + 2. Set up items, triggers, and actions for alerting. + +- **Example Configuration**: Sample Zabbix agent configurations are available in the `zabbix` folder. + +--- + +## How to Use + +1. Clone the repository and navigate to the `monitoring` folder. +2. Follow setup instructions in each subfolder for installing and configuring the respective monitoring tool. +3. Use the example configurations and scripts provided to integrate monitoring into your application. + +## Contributing + +Contributions are welcome! If you’d like to add more monitoring solutions or enhance existing configurations, please submit a pull request with a detailed description of your changes. diff --git a/monitoring/checkmk/checkmk_setup.md b/monitoring/checkmk/checkmk_setup.md new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/datadog/datadog_example.py b/monitoring/datadog/datadog_example.py new file mode 100644 index 0000000..5fbaaea --- /dev/null +++ b/monitoring/datadog/datadog_example.py @@ -0,0 +1,5 @@ +from ddtrace import tracer + +@tracer.wrap(name="custom_request_trace") +def process_request(): + time.sleep(random.random()) diff --git a/monitoring/datadog/datadog_setup.md b/monitoring/datadog/datadog_setup.md new file mode 100644 index 0000000..078e4e8 --- /dev/null +++ b/monitoring/datadog/datadog_setup.md @@ -0,0 +1,31 @@ +1. **Docker Compose**: Run Datadog Agent in Docker and configure it to monitor the Python app. + + **docker-compose.yml** + ```yaml + datadog: + image: datadog/agent:latest + environment: + - DD_API_KEY=your_datadog_api_key + - DD_LOGS_ENABLED=true + - DD_APM_ENABLED=true + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /proc/:/host/proc/:ro + - /sys/fs/cgroup:/host/sys/fs/cgroup:ro + ``` + +2. **Python Code**: Use `ddtrace` library for tracing requests. + + ```bash + pip install ddtrace + ``` + + **datadog_example.py** + ```python + from ddtrace import tracer + + @tracer.wrap(name="custom_request_trace") + def process_request(): + time.sleep(random.random()) + ``` + \ No newline at end of file diff --git a/monitoring/elasticsearch/elastic_log_integration.py b/monitoring/elasticsearch/elastic_log_integration.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/grafana/grafana_setup.md b/monitoring/grafana/grafana_setup.md new file mode 100644 index 0000000..e99eee0 --- /dev/null +++ b/monitoring/grafana/grafana_setup.md @@ -0,0 +1,21 @@ +1. **Docker Compose**: Use Docker Compose to run Grafana along with Prometheus. + + Add Grafana service to **docker-compose.yml** + ```yaml + grafana: + image: grafana/grafana + container_name: grafana + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + ``` + +2. **Grafana Setup**: After starting the Docker Compose setup, log in to Grafana at `http://localhost:3000` with default credentials (`admin/admin`). +3. **Add Data Source**: + - Go to **Configuration > Data Sources** in Grafana. + - Add Prometheus with the URL `http://prometheus:9090`. + +#### Process +1. Create dashboards in Grafana using the imported Prometheus metrics. +2. Set up alerts on critical metrics (e.g., request time or count). diff --git a/monitoring/influxdb/influxdb_integration.py b/monitoring/influxdb/influxdb_integration.py new file mode 100644 index 0000000..e69de29 diff --git a/monitoring/prometheus/prometheus_client_example.py b/monitoring/prometheus/prometheus_client_example.py new file mode 100644 index 0000000..25a57dc --- /dev/null +++ b/monitoring/prometheus/prometheus_client_example.py @@ -0,0 +1,13 @@ +from prometheus_client import start_http_server, Gauge +import random, time + +cpu_usage = Gauge("cpu_usage", "CPU usage in percent") + +def monitor_cpu(): + while True: + usage = random.uniform(10, 90) + cpu_usage.set(usage) + time.sleep(2) + +start_http_server(8000) +monitor_cpu() diff --git a/monitoring/prometheus/prometheus_details.md b/monitoring/prometheus/prometheus_details.md new file mode 100644 index 0000000..9fa1c3e --- /dev/null +++ b/monitoring/prometheus/prometheus_details.md @@ -0,0 +1,50 @@ +1. **Docker Compose**: Use Docker Compose to run Prometheus with Python. + + **docker-compose.yml** + ```yaml + version: '3.8' + services: + prometheus: + image: prom/prometheus + container_name: prometheus + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + ``` + +2. **Prometheus Configuration**: Define Prometheus scraping intervals and target services. + + **prometheus.yml** + ```yaml + global: + scrape_interval: 15s + + scrape_configs: + - job_name: 'python_app' + scrape_interval: 5s + static_configs: + - targets: ['host.docker.internal:8000'] + ``` + +3. **Python Code**: Use the `prometheus_client` library in Python to create and expose metrics. + + **prometheus_example.py** + ```python + from prometheus_client import start_http_server, Summary, Counter + import time + import random + + REQUEST_TIME = Summary('request_processing_seconds', 'Time spent processing request') + REQUEST_COUNT = Counter('request_count', 'Total number of requests') + + @REQUEST_TIME.time() + def process_request(): + REQUEST_COUNT.inc() + time.sleep(random.random()) + + if __name__ == '__main__': + start_http_server(8000) + while True: + process_request() + ``` diff --git a/monitoring/sentry/sentry_integration.py b/monitoring/sentry/sentry_integration.py new file mode 100644 index 0000000..bf0889f --- /dev/null +++ b/monitoring/sentry/sentry_integration.py @@ -0,0 +1,25 @@ +import sentry_sdk +from flask import Flask + +# Initialize Sentry +sentry_sdk.init( + dsn="https://your-sentry-dsn", + traces_sample_rate=1.0 +) + +app = Flask(__name__) + + +@app.route('/') +def home(): + return "Hello, World!" + + +@app.route('/error') +def trigger_error(): + division_by_zero = 1 / 0 + return "Error triggered" + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/monitoring/sentry/sentry_setup.md b/monitoring/sentry/sentry_setup.md new file mode 100644 index 0000000..d35c778 --- /dev/null +++ b/monitoring/sentry/sentry_setup.md @@ -0,0 +1,33 @@ +1. **Docker Compose**: Sentry can also be self-hosted with Docker Compose, but here we’ll focus on integrating Sentry with Python. + +2. **Python Code**: Use the `sentry-sdk` to capture errors. + + **sentry_example.py** + ```python + import sentry_sdk + from flask import Flask + + # Initialize Sentry + sentry_sdk.init( + dsn="https://your-sentry-dsn", + traces_sample_rate=1.0 + ) + + app = Flask(__name__) + + @app.route('/') + def home(): + return "Hello, World!" + + @app.route('/error') + def trigger_error(): + division_by_zero = 1 / 0 + return "Error triggered" + + if __name__ == "__main__": + app.run(debug=True) + ``` + +#### Process +1. Run `sentry_example.py` and navigate to `http://localhost:5000/error` to trigger an error. +2. Verify that the error appears in your Sentry project dashboard. diff --git a/monitoring/zabbix/zabbix_api_example.py b/monitoring/zabbix/zabbix_api_example.py new file mode 100644 index 0000000..e69de29 diff --git a/payments/README.md b/payments/README.md new file mode 100644 index 0000000..9c45e1f --- /dev/null +++ b/payments/README.md @@ -0,0 +1,235 @@ +# Payments + +This folder contains Python scripts for integrating various payment providers, including traditional gateways, local payment methods, and cryptocurrency payments. Each script provides functions for initiating and managing transactions with different payment providers. + +## Contents + +- [Braintree](#braintree) +- [BTCPay](#btcpay) +- [Chargebee](#chargebee) +- [GetLago](#getlago) +- [M-Pesa](#m-pesa) +- [MTN MoMo](#mtn-momo) +- [Orange Money](#orange-money) +- [PayPal](#paypal) +- [Paystack](#paystack) +- [Razorpay](#razorpay) +- [Square](#square) +- [Stripe](#stripe) + +--- + +### Braintree + +**Braintree** is a full-stack payment platform that makes it easy to accept payments online. + +- **File**: `braintree_payment.py` +- **Setup**: + 1. Install the Braintree SDK: `pip install braintree`. + 2. Configure Braintree credentials (merchant ID, public key, private key). + +- **Example Usage**: + ```python + from braintree_payment import create_transaction + create_transaction(100, "USD", "payment_method_nonce") + ``` + +--- + +### BTCPay + +**BTCPay** is a cryptocurrency payment processor that allows businesses to accept Bitcoin and other cryptocurrencies. + +- **File**: `btcpay_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Set up BTCPay server and API keys. + +- **Example Usage**: + ```python + from btcpay_payment import create_invoice + create_invoice(0.01, "BTC") + ``` + +--- + +### Chargebee + +**Chargebee** is a subscription management and recurring billing software for SaaS businesses. + +- **File**: `chargebee_payment.py` +- **Setup**: + 1. Install Chargebee Python library: `pip install chargebee`. + 2. Configure API key and site name. + +- **Example Usage**: + ```python + from chargebee_payment import create_subscription + create_subscription("plan_id", "customer_id") + ``` + +--- + +### GetLago + +**GetLago** is an open-source billing API for managing subscription billing and usage-based billing. + +- **File**: `getlago_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Configure API key. + +- **Example Usage**: + ```python + from getlago_payment import create_invoice + create_invoice("customer_id", 2000, "USD") + ``` + +--- + +### M-Pesa + +**M-Pesa** is a mobile money service that allows users to transfer money and make payments. + +- **File**: `mpesa_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Configure M-Pesa API credentials. + +- **Example Usage**: + ```python + from mpesa_payment import initiate_payment + initiate_payment("254700000000", 1000) + ``` + +--- + +### MTN MoMo + +**MTN MoMo** provides mobile money services that allow users to make payments and transfers using mobile devices. + +- **File**: `mtn_momo_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Configure MTN MoMo API credentials. + +- **Example Usage**: + ```python + from mtn_momo_payment import request_payment + request_payment("256700000000", 500) + ``` + +--- + +### Orange Money + +**Orange Money** is a mobile money service that allows users to send and receive money, pay bills, and more. + +- **File**: `orange_money_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Configure Orange Money API credentials. + +- **Example Usage**: + ```python + from orange_money_payment import initiate_transaction + initiate_transaction("237600000000", 1500) + ``` + +--- + +### PayPal + +**PayPal** is a widely used online payment platform for individuals and businesses. + +- **File**: `paypal_payment.py` +- **Setup**: + 1. Install PayPal SDK: `pip install paypalrestsdk`. + 2. Configure PayPal client ID and secret. + +- **Example Usage**: + ```python + from paypal_payment import create_payment + create_payment(50, "USD") + ``` + +--- + +### Paystack + +**Paystack** is a Nigerian payment gateway that makes it easy for businesses in Africa to accept payments. + +- **File**: `paystack_payment.py` +- **Setup**: + 1. Install requests: `pip install requests`. + 2. Configure Paystack API key. + +- **Example Usage**: + ```python + from paystack_payment import initialize_transaction + initialize_transaction(10000, "NGN", "user@example.com") + ``` + +--- + +### Razorpay + +**Razorpay** provides payment solutions for businesses to accept, process, and disburse payments. + +- **File**: `razorpay_payment.py` +- **Setup**: + 1. Install Razorpay SDK: `pip install razorpay`. + 2. Configure Razorpay key ID and secret. + +- **Example Usage**: + ```python + from razorpay_payment import create_order + create_order(50000, "INR") + ``` + +--- + +### Square + +**Square** is a popular payment platform for accepting payments, managing invoices, and processing transactions. + +- **File**: `squareup_payment.py` +- **Setup**: + 1. Install Square SDK: `pip install squareup`. + 2. Configure Square access token. + +- **Example Usage**: + ```python + from squareup_payment import create_payment + create_payment(100, "USD", "nonce_from_card") + ``` + +--- + +### Stripe + +**Stripe** is a payment processor for online and in-person payments, supporting various currencies and payment methods. + +- **File**: `stripe_payment.py` +- **Setup**: + 1. Install Stripe SDK: `pip install stripe`. + 2. Configure Stripe API key. + +- **Example Usage**: + ```python + from stripe_payment import charge_card + charge_card(2000, "usd", "card_token") + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `payments` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your credentials for each payment provider. +4. Run the scripts to initiate transactions with the specified payment provider. + +## Contributing + +Contributions are welcome! If you’d like to add more payment providers or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/payments/braintree_payment.py b/payments/braintree_payment.py new file mode 100644 index 0000000..cf88b60 --- /dev/null +++ b/payments/braintree_payment.py @@ -0,0 +1,46 @@ +import os + +import braintree +from dotenv import load_dotenv + +load_dotenv() + +gateway = braintree.BraintreeGateway( + braintree.Configuration( + braintree.Environment.Sandbox, + merchant_id=os.getenv('BRAINTREE_MERCHANT_ID'), + public_key=os.getenv('BRAINTREE_PUBLIC_KEY'), + private_key=os.getenv('BRAINTREE_PRIVATE_KEY') + ) +) + + +def create_braintree_payment(amount, payment_method_nonce): + """ + Create a payment with Braintree. + + Args: + amount (str): Payment amount. + payment_method_nonce (str): Nonce from the payment method. + + Returns: + dict: Braintree payment response. + """ + result = gateway.transaction.sale({ + "amount": amount, + "payment_method_nonce": payment_method_nonce, + "options": {"submit_for_settlement": True} + }) + + if result.is_success: + print(f"Payment successful! Transaction ID: {result.transaction.id}") + return result.transaction + else: + print(f"Payment failed: {result.message}") + return None + + +# Example usage +if __name__ == "__main__": + transaction = create_braintree_payment('10.00', 'fake-valid-nonce') + print(transaction) diff --git a/payments/btcpay_payment.py b/payments/btcpay_payment.py new file mode 100644 index 0000000..e577bfc --- /dev/null +++ b/payments/btcpay_payment.py @@ -0,0 +1,46 @@ +import requests +import os + +from dotenv import load_dotenv + +load_dotenv() + +BTCPAY_API_KEY = os.getenv('BTCPAY_API_KEY') +BTCPAY_BASE_URL = "https://your-btcpay-server.com" + + +def create_btcpay_invoice(amount, currency, buyer_email): + """ + Create an invoice in BTCPay Server. + + Args: + amount (float): The amount to charge. + currency (str): Currency (e.g., "USD", "BTC"). + buyer_email (str): Buyer's email. + + Returns: + dict: BTCPay invoice response. + """ + headers = { + "Authorization": f"token {BTCPAY_API_KEY}", + "Content-Type": "application/json" + } + + data = { + "price": amount, + "currency": currency, + "buyerEmail": buyer_email + } + + response = requests.post(f"{BTCPAY_BASE_URL}/api/v1/invoices", json=data, headers=headers) + return response.json() + + +# Example usage +if __name__ == "__main__": + invoice = create_btcpay_invoice( + 0.001, + "BTC", + "btc_buyer@example.com", + ) + print(invoice) diff --git a/payments/chargebee_payment.py b/payments/chargebee_payment.py new file mode 100644 index 0000000..76b806e --- /dev/null +++ b/payments/chargebee_payment.py @@ -0,0 +1,100 @@ +import os + +import chargebee +from dotenv import load_dotenv + +load_dotenv() + +CHARGEBEE_SITE = os.getenv('CHARGEBEE_SITE') # Here you puy your Chargebee site identifier +CHARGEBEE_API_KEY = os.getenv('CHARGEBEE_API_KEY') + +chargebee.configure(CHARGEBEE_API_KEY, CHARGEBEE_SITE) + + +def create_chargebee_subscription(customer_id, plan_id): + """ + Create a subscription in Chargebee for a customer. + + Args: + customer_id (str): The unique customer ID. + plan_id (str): The ID of the plan to subscribe the customer to. + + Returns: + dict: Chargebee subscription response. + """ + try: + result = chargebee.Subscription.create({ + "plan_id": plan_id, + "customer": { + "id": customer_id + } + }) + print(f"Subscription created successfully: {result['subscription']['id']}") + return result + except chargebee.APIError as e: + print(f"Error creating subscription: {e}") + return None + + +# Example usage (create a new subscription) +if __name__ == "__main__": + subscription = create_chargebee_subscription("customer_12345", "premium_plan") + print(subscription) + + +def create_chargebee_customer(email, first_name, last_name): + """ + Create a new customer in Chargebee. + + Args: + email (str): The customer's email address. + first_name (str): Customer's first name. + last_name (str): Customer's last name. + + Returns: + dict: Chargebee customer response. + """ + try: + result = chargebee.Customer.create({ + "email": email, + "first_name": first_name, + "last_name": last_name + }) + print(f"Customer created successfully: {result['customer']['id']}") + return result + except chargebee.APIError as e: + print(f"Error creating customer: {e}") + return None + + +# Example usage (create a new customer) +if __name__ == "__main__": + customer = create_chargebee_customer("customer@example.com", "John", "Doe") + print(customer) + + +def cancel_chargebee_subscription(subscription_id): + """ + Cancel a subscription in Chargebee. + + Args: + subscription_id (str): The ID of the subscription to cancel. + + Returns: + dict: Chargebee subscription cancellation response. + """ + try: + result = chargebee.Subscription.cancel(subscription_id, { + "end_of_term": True + }) + print(f"Subscription canceled: {result['subscription']['id']}") + return result + except chargebee.APIError as e: + print(f"Error canceling subscription: {e}") + return None + + +# Example usage (cancel a subscription) +if __name__ == "__main__": + cancellation = cancel_chargebee_subscription("sub_12345") + print(cancellation) diff --git a/payments/getlago_payment.py b/payments/getlago_payment.py new file mode 100644 index 0000000..984eddb --- /dev/null +++ b/payments/getlago_payment.py @@ -0,0 +1,48 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +LAGO_API_KEY = os.getenv('LAGO_API_KEY') +LAGO_BASE_URL = "https://api.getlago.com" + + +def create_customer(customer_id, email, name): + """ + Create a customer in GetLago. + + Args: + customer_id (str): The unique ID for the customer. + email (str): Customer's email. + name (str): Customer's name. + + Returns: + dict: GetLago customer response. + """ + headers = { + "Authorization": f"Bearer {LAGO_API_KEY}", + "Content-Type": "application/json" + } + + data = { + "customer": { + "external_id": customer_id, + "email": email, + "name": name + } + } + + response = requests.post(f"{LAGO_BASE_URL}/v1/customers", json=data, headers=headers) + return response.json() + + +# Example usage +if __name__ == "__main__": + customer = create_customer( + "cust_1234", + "customer@example.com", + "John Doe", + ) + print(customer) diff --git a/payments/mpesa_payment.py b/payments/mpesa_payment.py new file mode 100644 index 0000000..a44c981 --- /dev/null +++ b/payments/mpesa_payment.py @@ -0,0 +1,55 @@ +import os + +import requests +from requests.auth import HTTPBasicAuth + +MPESA_CONSUMER_KEY = os.getenv('MPESA_CONSUMER_KEY') +MPESA_CONSUMER_SECRET = os.getenv('MPESA_CONSUMER_SECRET') +MPESA_SHORTCODE = os.getenv('MPESA_SHORTCODE') +MPESA_PASSKEY = os.getenv('MPESA_PASSKEY') + + +def get_mpesa_access_token(): + """ + Retrieve the access token from M-Pesa API. + """ + url = "https://sandbox.safaricom.co.ke/oauth/v1/generate?grant_type=client_credentials" + response = requests.get(url, auth=HTTPBasicAuth(MPESA_CONSUMER_KEY, MPESA_CONSUMER_SECRET)) + access_token = response.json()['access_token'] + return access_token + + +def mpesa_stk_push(phone_number, amount, account_reference, transaction_desc): + """ + Perform an STK Push (Sim Tool Kit) payment request using M-Pesa. + """ + access_token = get_mpesa_access_token() + api_url = "https://sandbox.safaricom.co.ke/mpesa/stkpush/v1/processrequest" + headers = {"Authorization": f"Bearer {access_token}"} + payload = { + "BusinessShortCode": MPESA_SHORTCODE, + "Password": "your_password_here", + "Timestamp": "20240101010101", + "TransactionType": "CustomerPayBillOnline", + "Amount": amount, + "PartyA": phone_number, + "PartyB": MPESA_SHORTCODE, + "PhoneNumber": phone_number, + "CallBackURL": "https://your-callback-url.com", + "AccountReference": account_reference, + "TransactionDesc": transaction_desc + } + + payment_response = requests.post(api_url, json=payload, headers=headers) + return payment_response.json() + + +# Example usage +if __name__ == "__main__": + response = mpesa_stk_push( + '254712345678', + 1000, + 'TestRef123', + 'Test Payment', + ) + print(response) diff --git a/payments/mtn_momo_payment.py b/payments/mtn_momo_payment.py new file mode 100644 index 0000000..bca60ca --- /dev/null +++ b/payments/mtn_momo_payment.py @@ -0,0 +1,53 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +MOMO_API_USER = os.getenv('MOMO_API_USER') +MOMO_API_KEY = os.getenv('MOMO_API_KEY') +MOMO_ENVIRONMENT = os.getenv('MOMO_ENVIRONMENT', 'sandbox') # or 'live' when in production + + +def get_momo_access_token(): + """ + Get an access token from MTN MoMo API. + """ + url = f"https://{MOMO_ENVIRONMENT}.momoapi.mtn.com/collection/token/" + headers = {"Authorization": f"Basic {MOMO_API_KEY}"} + momo_token_response = requests.post(url, headers=headers) + return momo_token_response.json()['access_token'] + + +def momo_request_payment(phone_number, amount, external_id, currency="EUR"): + """ + Request a payment via MTN MoMo. + """ + access_token = get_momo_access_token() + api_url = f"https://{MOMO_ENVIRONMENT}.momoapi.mtn.com/collection/v1_0/requesttopay" + headers = {"Authorization": f"Bearer {access_token}"} + payload = { + "amount": str(amount), + "currency": currency, + "externalId": external_id, + "payer": { + "partyIdType": "MSISDN", + "partyId": phone_number + }, + "payerMessage": "Payment for services", + "payeeNote": "Service payment" + } + + payment_response = requests.post(api_url, json=payload, headers=headers) + return payment_response.json() + + +# Example usage +if __name__ == "__main__": + response = momo_request_payment( + '256712345678', + 1000, + 'ext12345', + ) + print(response) diff --git a/payments/orange_money_payment.py b/payments/orange_money_payment.py new file mode 100644 index 0000000..06ed2f3 --- /dev/null +++ b/payments/orange_money_payment.py @@ -0,0 +1,65 @@ +import requests +import os +from dotenv import load_dotenv + +load_dotenv() + +ORANGE_APP_ID = os.getenv('ORANGE_APP_ID') +ORANGE_APP_SECRET = os.getenv('ORANGE_APP_SECRET') + +def get_orange_access_token(): + """ + Get an access token from Orange Money API. + """ + url = "https://api.orange.com/oauth/v3/token" + headers = { + "Authorization": f"Basic {ORANGE_APP_ID}:{ORANGE_APP_SECRET}", + "Content-Type": "application/x-www-form-urlencoded" + } + payload = {"grant_type": "client_credentials"} + orange_token_response = requests.post(url, headers=headers, data=payload) + return orange_token_response.json()['access_token'] + + +def orange_money_request_payment(phone_number, amount, order_id, currency="XAF"): + """ + Request a payment via Orange Money. + + Args: + phone_number (str): The payer's phone number. + amount (int): The payment amount. + order_id (str): A unique identifier for the payment. + currency (str): The currency code, default is XAF. + + Returns: + dict: Orange Money payment response. + """ + access_token = get_orange_access_token() + api_url = "https://api.orange.com/orange-money-webpay/dev/v1/webpayment" + headers = {"Authorization": f"Bearer {access_token}"} + + payload = { + "merchant_key": "your_merchant_key", + "amount": amount, + "currency": currency, + "order_id": order_id, + "return_url": "https://your-return-url.com", + "cancel_url": "https://your-cancel-url.com", + "notif_url": "https://your-notif-url.com", + "lang": "en", + "reference": "Test Payment", + "payer": {"id_type": "MSISDN", "id": phone_number} + } + + om_request_response = requests.post(api_url, json=payload, headers=headers) + return om_request_response.json() + + +# Example usage +if __name__ == "__main__": + response = orange_money_request_payment( + 'XXXXXXXXXXXXX', + 5000, + 'order12345' + ) + print(response) diff --git a/payments/paypal_payment.py b/payments/paypal_payment.py new file mode 100644 index 0000000..6afbf12 --- /dev/null +++ b/payments/paypal_payment.py @@ -0,0 +1,50 @@ +import paypalrestsdk +import os +from dotenv import load_dotenv + +load_dotenv() + +paypalrestsdk.configure({ + "mode": "sandbox", # Use "live" for production + "client_id": os.getenv('PAYPAL_CLIENT_ID'), + "client_secret": os.getenv('PAYPAL_CLIENT_SECRET') +}) + +def create_paypal_payment(total, currency, description, return_url, cancel_url): + """ + Create a PayPal payment. + + Args: + total (str): Payment amount. + currency (str): Currency code (e.g., 'USD'). + description (str): Description of the payment. + return_url (str): URL to redirect after successful payment. + cancel_url (str): URL to redirect after a failed or canceled payment. + + Returns: + dict: PayPal payment response. + """ + paypal_payment = paypalrestsdk.Payment({ + "intent": "sale", + "payer": {"payment_method": "paypal"}, + "transactions": [{"amount": {"total": total, "currency": currency}, "description": description}], + "redirect_urls": {"return_url": return_url, "cancel_url": cancel_url} + }) + + if paypal_payment.create(): + print(f"Payment created successfully: {paypal_payment.id}") + return paypal_payment + else: + print(paypal_payment.error) + return None + +# Example usage +if __name__ == "__main__": + payment = create_paypal_payment( + '10.00', + 'USD', + 'Test Payment from PayPal', + 'https://your-return-url.com', + 'https://your-cancel-url.com', + ) + print(payment) diff --git a/payments/paystack_payment.py b/payments/paystack_payment.py new file mode 100644 index 0000000..ef34fec --- /dev/null +++ b/payments/paystack_payment.py @@ -0,0 +1,45 @@ +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() + +PAYSTACK_SECRET_KEY = os.getenv('PAYSTACK_SECRET_KEY') + + +def create_paystack_payment(amount, email, reference): + """ + Create a payment request with Paystack. + + Args: + amount (int): The payment amount in kobo (for NGN). + email (str): The customer's email. + reference (str): Unique transaction reference. + + Returns: + dict: Paystack payment response. + """ + headers = { + "Authorization": f"Bearer {PAYSTACK_SECRET_KEY}", + "Content-Type": "application/json" + } + + data = { + "email": email, + "amount": amount, # In kobo for NGN + "reference": reference + } + + response = requests.post("https://api.paystack.co/transaction/initialize", json=data, headers=headers) + return response.json() + + +# Example usage +if __name__ == "__main__": + payment = create_paystack_payment( + 50000, + "customer@example.com", + "tx_ref_001", + ) + print(payment) diff --git a/payments/razorpay_payment.py b/payments/razorpay_payment.py new file mode 100644 index 0000000..76d0007 --- /dev/null +++ b/payments/razorpay_payment.py @@ -0,0 +1,42 @@ +import os +import uuid +import razorpay +from dotenv import load_dotenv + +load_dotenv() + +RAZORPAY_KEY_ID = os.getenv('RAZORPAY_KEY_ID') +RAZORPAY_KEY_SECRET = os.getenv('RAZORPAY_KEY_SECRET') + +client = razorpay.Client(auth=(RAZORPAY_KEY_ID, RAZORPAY_KEY_SECRET)) + + +def create_razorpay_payment(amount, currency): + """ + Create a payment with Razorpay. + + Args: + amount (int): Payment amount in the smallest currency unit (e.g., paise for INR). + currency (str): Currency code (e.g., 'INR'). + + Returns: + dict: Razorpay payment response. + """ + payment = client.order.create({ + "amount": amount, + "currency": currency, + "receipt": str(uuid.uuid4()), + "payment_capture": "1" + }) + + print(f"Payment order created: {payment['id']}") + return payment + + +# Example usage +if __name__ == "__main__": + order = create_razorpay_payment( + 50000, + 'USD', + ) + print(order) diff --git a/payments/squareup_payment.py b/payments/squareup_payment.py new file mode 100644 index 0000000..0db2e61 --- /dev/null +++ b/payments/squareup_payment.py @@ -0,0 +1,48 @@ +from square.client import Client +import os +import uuid + +from dotenv import load_dotenv + +load_dotenv() + +SQUARE_ACCESS_TOKEN = os.getenv('SQUARE_ACCESS_TOKEN') + +client = Client(access_token=SQUARE_ACCESS_TOKEN) + +def create_square_payment(amount, currency, source_id, location_id): + """ + Create a payment with Square. + + Args: + amount (int): Payment amount in the smallest currency unit (e.g., cents for USD). + currency (str): Currency code (e.g., 'USD'). + source_id (str): Source token or card. + location_id (str): Square location ID. + + Returns: + dict: Square payment response. + """ + response = client.payments.create_payment({ + "source_id": source_id, + "amount_money": {"amount": amount, "currency": currency}, + "idempotency_key": str(uuid.uuid4()), + "location_id": location_id + }) + + if response.is_success(): + print(f"Payment successful! Payment ID: {response.body['payment']['id']}") + return response.body + else: + print(f"Payment failed: {response.errors}") + return None + +# Example usage +if __name__ == "__main__": + payment = create_square_payment( + 5000, + 'USD', + 'source-id-token', + 'location-id', + ) + print(payment) diff --git a/payments/stripe_payment.py b/payments/stripe_payment.py new file mode 100644 index 0000000..8732821 --- /dev/null +++ b/payments/stripe_payment.py @@ -0,0 +1,41 @@ +import stripe +import os + +STRIPE_SECRET_KEY = os.getenv('STRIPE_SECRET_KEY') +stripe.api_key = STRIPE_SECRET_KEY + +def create_stripe_payment(amount, currency, source, description): + """ + Create a payment with Stripe. + + Args: + amount (int): Amount to charge (in the smallest unit of the currency, e.g., cents for USD). + currency (str): Currency code (e.g., 'usd'). + source (str): Source token or card. + description (str): Description of the payment. + + Returns: + dict: Stripe payment response. + """ + try: + stripe_charge = stripe.Charge.create( + amount=amount, + currency=currency, + source=source, + description=description + ) + print(f"Payment successful! Charge ID: {stripe_charge.id}") + return stripe_charge + except stripe.error.StripeError as e: + print(f"Payment failed: {e.error.message}") + return None + +# Example usage +if __name__ == "__main__": + charge = create_stripe_payment( + 5000, + 'usd', + 'tok_visa', + 'Test Payment from Stripe', + ) + print(charge) diff --git a/queueing/README.md b/queueing/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/queueing/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/scaling/README.md b/scaling/README.md new file mode 100644 index 0000000..d4edfd0 --- /dev/null +++ b/scaling/README.md @@ -0,0 +1 @@ +Coming Soon... \ No newline at end of file diff --git a/security/README.md b/security/README.md new file mode 100644 index 0000000..b21c7be --- /dev/null +++ b/security/README.md @@ -0,0 +1,202 @@ +# Security + +This folder contains various security implementations, from encryption and password hashing to secure authentication and rate limiting. Each file provides a specific security feature commonly used in backend applications, helping protect data, manage access control, and secure communications. + +## Contents + +- [AES Encryption](#aes-encryption) +- [Password Hashing with Bcrypt](#password-hashing-with-bcrypt) +- [CORS (Cross-Origin Resource Sharing)](#cors-cross-origin-resource-sharing) +- [CSRF Protection](#csrf-protection) +- [Flask Security Headers](#flask-security-headers) +- [Flask Rate Limiting](#flask-rate-limiting) +- [JWT Authentication](#jwt-authentication) +- [OTP (One-Time Password)](#otp-one-time-password) +- [Passkeys](#passkeys) +- [RBAC and ABAC](#rbac-and-abac) +- [Apache SSL Configuration](#apache-ssl-configuration) +- [TLS Configuration for Nginx](#tls-configuration-for-nginx) + +--- + +### AES Encryption + +**AES (Advanced Encryption Standard)** is a symmetric encryption algorithm used for securing sensitive data. + +- **File**: `aes_encryption.py` +- **Setup**: + 1. Install `cryptography` library: `pip install cryptography`. + +- **Example Usage**: + ```python + from aes_encryption import encrypt_data, decrypt_data + encrypted = encrypt_data("my_secret_data") + decrypted = decrypt_data(encrypted) + ``` + +--- + +### Password Hashing with Bcrypt + +**Bcrypt** is a cryptographic algorithm for hashing passwords, making them difficult to reverse. + +- **File**: `bcrypt_passwords.py` +- **Setup**: + 1. Install `bcrypt` library: `pip install bcrypt`. + +- **Example Usage**: + ```python + from bcrypt_passwords import hash_password, check_password + hashed = hash_password("my_password") + is_valid = check_password("my_password", hashed) + ``` + +--- + +### CORS (Cross-Origin Resource Sharing) + +**CORS** controls the domains that can access your API, adding a layer of security to prevent unauthorized cross-origin requests. + +- **File**: `cors.py` + +- **Example Usage**: + ```python + from cors import setup_cors + setup_cors(app) + ``` + +--- + +### CSRF Protection + +**CSRF (Cross-Site Request Forgery)** prevents malicious users from making requests on behalf of other users without their consent. + +- **File**: `csrf.py` + +- **Example Usage**: + ```python + from csrf import generate_csrf_token, verify_csrf_token + token = generate_csrf_token() + is_valid = verify_csrf_token(token) + ``` + +--- + +### Flask Security Headers + +**Security Headers** in Flask provide an extra layer of security by adding headers to responses, protecting against vulnerabilities like clickjacking and XSS. + +- **File**: `flask_headers.py` + +- **Example Usage**: + ```python + from flask_headers import set_security_headers + set_security_headers(app) + ``` + +--- + +### Flask Rate Limiting + +**Rate Limiting** restricts the number of requests an IP or user can make, reducing the risk of abuse and denial-of-service attacks. + +- **File**: `flask_rate_limiting.py` + +- **Example Usage**: + ```python + from flask_rate_limiting import apply_rate_limit + apply_rate_limit(app) + ``` + +--- + +### JWT Authentication + +**JWT (JSON Web Token)** is a method of secure token-based authentication, commonly used in APIs. + +- **File**: `jwt_auth.py` + +- **Example Usage**: + ```python + from jwt_auth import create_jwt, verify_jwt + token = create_jwt({"user_id": 123}) + is_valid = verify_jwt(token) + ``` + +--- + +### OTP (One-Time Password) + +**OTP** provides a single-use password, usually as a secondary authentication factor. + +- **File**: `otp.py` + +- **Example Usage**: + ```python + from otp import generate_otp, verify_otp + otp_code = generate_otp() + is_valid = verify_otp(otp_code) + ``` + +--- + +### Passkeys + +**Passkeys** offer a secure, passwordless way of authenticating users, helping prevent credential-based attacks. + +- **File**: `passkeys.py` + +- **Example Usage**: + ```python + from passkeys import create_passkey, verify_passkey + passkey = create_passkey("user_id") + is_valid = verify_passkey("user_id", passkey) + ``` + +--- + +### RBAC and ABAC + +**RBAC (Role-Based Access Control)** and **ABAC (Attribute-Based Access Control)** provide granular control over resource access based on user roles or attributes. + +- **File**: `rbac_abac.py` + +- **Example Usage**: + ```python + from rbac_abac import check_rbac, check_abac + has_access = check_rbac("user_role", "resource") + has_attr_access = check_abac({"role": "admin"}, "resource") + ``` + +--- + +### Apache SSL Configuration + +**SSL Configuration** for Apache provides secure HTTPS communication for web servers, ensuring data is encrypted between the client and server. + +- **File**: `apache_setup.md` + +- **Usage**: + - This file provides instructions for setting up SSL on Apache using a Let's Encrypt certificate. Follow the steps to secure your Apache server with SSL. + +--- + +### TLS Configuration for Nginx + +**TLS Configuration** in Nginx secures communication over HTTPS and enforces best practices for strong ciphers and protocol settings. + +- **File**: `tls_nginx_security.conf` +- **Usage**: + - Configure this file in your Nginx server block to enforce strong TLS settings, improving security for your web server. + +--- + +## How to Use + +1. Clone the repository and navigate to the `security` folder. +2. Install any required dependencies as mentioned in each section. +3. Refer to each script or configuration file for setup and usage instructions. + +## Contributing + +Contributions are welcome! If you’d like to add more security features or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/security/aes_encryption.py b/security/aes_encryption.py new file mode 100644 index 0000000..9a65c1f --- /dev/null +++ b/security/aes_encryption.py @@ -0,0 +1,26 @@ +from Crypto.Cipher import AES +from Crypto.Util.Padding import pad, unpad +from base64 import b64encode, b64decode +import os + +KEY = os.getenv('AES_SECRET_KEY')[:32].encode() + +def encrypt(plain_text): + cipher = AES.new(KEY, AES.MODE_CBC) + iv = cipher.iv + cipher_text = cipher.encrypt(pad(plain_text.encode(), AES.block_size)) + return b64encode(iv + cipher_text).decode('utf-8') + +def decrypt(cipher_text): + cipher_text = b64decode(cipher_text) + iv = cipher_text[:16] + cipher = AES.new(KEY, AES.MODE_CBC, iv) + plain_text = unpad(cipher.decrypt(cipher_text[16:]), AES.block_size) + return plain_text.decode('utf-8') + +# Example usage +if __name__ == "__main__": + encrypted = encrypt("This is a secret message") + print(f"Encrypted: {encrypted}") + decrypted = decrypt(encrypted) + print(f"Decrypted: {decrypted}") diff --git a/security/apache_setup.md b/security/apache_setup.md new file mode 100644 index 0000000..63b47f1 --- /dev/null +++ b/security/apache_setup.md @@ -0,0 +1,64 @@ +To set up SSL for Apache, you can use the following configuration. This example assumes you have SSL certificates from Let's Encrypt, but it can be adapted for other certificate providers. + +```apache + + ServerName yourdomain.com + DocumentRoot /var/www/html + + # SSL Configuration + SSLEngine on + SSLCertificateFile /etc/letsencrypt/live/yourdomain.com/fullchain.pem + SSLCertificateKeyFile /etc/letsencrypt/live/yourdomain.com/privkey.pem + SSLCertificateChainFile /etc/letsencrypt/live/yourdomain.com/chain.pem + + # Recommended SSL Settings + SSLProtocol all -SSLv2 -SSLv3 + SSLCipherSuite HIGH:!aNULL:!MD5 + SSLHonorCipherOrder on + + # Optional: Redirect HTTP to HTTPS + + RewriteEngine On + RewriteCond %{HTTPS} !=on + RewriteRule ^/?(.*) https://%{SERVER_NAME}/$1 [R,L] + + + # Proxy Settings (if needed to forward to an app server) + ProxyPreserveHost On + ProxyPass / http://localhost:8000/ + ProxyPassReverse / http://localhost:8000/ + + +# Redirect HTTP to HTTPS + + ServerName yourdomain.com + RewriteEngine On + RewriteCond %{HTTPS} off + RewriteRule ^/?(.*) https://%{SERVER_NAME}/$1 [R=301,L] + +``` + +### Explanation + +- `ServerName`: Specifies the domain name. +- `SSLCertificateFile`, `SSLCertificateKeyFile`, and `SSLCertificateChainFile`: Paths to the SSL certificate files provided by Let's Encrypt or another certificate provider. +- `SSLProtocol` and `SSLCipherSuite`: Restricts SSL/TLS protocols and cipher suites for enhanced security. +- `ProxyPass` and `ProxyPassReverse`: Forwards requests to an application running on `http://localhost:8000`. +- The HTTP `VirtualHost` listens on port 80 and redirects all requests to HTTPS. + +### Enable SSL and Restart Apache + +1. Enable the SSL module: + ```bash + sudo a2enmod ssl + ``` +2. Enable proxy modules if using a proxy: + ```bash + sudo a2enmod proxy proxy_http rewrite + ``` +3. Restart Apache: + ```bash + sudo systemctl restart apache2 + ``` + +This configuration provides SSL termination with Apache, redirecting all HTTP traffic to HTTPS and forwarding requests to an application server running on port 8000 if needed. diff --git a/security/bcrypt_passwords.py b/security/bcrypt_passwords.py new file mode 100644 index 0000000..33e0a21 --- /dev/null +++ b/security/bcrypt_passwords.py @@ -0,0 +1,16 @@ +import bcrypt + +def hash_password(user_password): + salt = bcrypt.gensalt() + hashed_password = bcrypt.hashpw(user_password.encode('utf-8'), salt) + return hashed_password + +def verify_password(stored_password, provided_password): + return bcrypt.checkpw(provided_password.encode('utf-8'), stored_password) + +# Example usage +if __name__ == "__main__": + password = "SecurePass123" + hashed = hash_password(password) + print(f"Hashed Password: {hashed}") + print(f"Password Verified: {verify_password(hashed, 'SecurePass123')}") diff --git a/security/cors.py b/security/cors.py new file mode 100644 index 0000000..5356c7d --- /dev/null +++ b/security/cors.py @@ -0,0 +1,24 @@ +from flask import Flask +from flask_cors import CORS +from starlette.middleware.cors import CORSMiddleware + +app = Flask(__name__) +CORS( + app, + middleware=CORSMiddleware, + resources={ + r"/api/*": { + "origins": "https://your-allowed-origin.com", + "methods": ["GET", "POST"], + } + } +) + + +@app.route('/api/data') +def api_data(): + return {"data": "This is secure data"} + + +if __name__ == "__main__": + app.run() diff --git a/security/csrf.py b/security/csrf.py new file mode 100644 index 0000000..0190448 --- /dev/null +++ b/security/csrf.py @@ -0,0 +1,24 @@ +from django.http import HttpResponse +from django.views.decorators.csrf import csrf_exempt +from flask import Flask, render_template, request +from flask_wtf.csrf import CSRFProtect + +app = Flask(__name__) +app.secret_key = 'your_secret_key' +csrf = CSRFProtect(app) + + +@app.route('/form', methods=['GET', 'POST']) +def form(): + if request.method == 'POST': + pass + return render_template('wtf_template.html') + + +@csrf_exempt +def my_view(request): + return HttpResponse("This view is CSRF exempt") + + +if __name__ == "__main__": + app.run() diff --git a/security/flask_headers.py b/security/flask_headers.py new file mode 100644 index 0000000..432a118 --- /dev/null +++ b/security/flask_headers.py @@ -0,0 +1,12 @@ +from flask import Flask +from flask_talisman import Talisman + +app = Flask(__name__) +Talisman(app) + +@app.route("/") +def home(): + return "Welcome to a secure site!" + +if __name__ == "__main__": + app.run() diff --git a/security/flask_rate_limiting.py b/security/flask_rate_limiting.py new file mode 100644 index 0000000..3e2514a --- /dev/null +++ b/security/flask_rate_limiting.py @@ -0,0 +1,22 @@ +from flask import Flask + +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address + +app = Flask(__name__) +limiter = Limiter( + key_func=get_remote_address, + app=app, + default_limits=["200 per day", "50 per hour"], + storage_uri="memory://", +) + + +@app.route("/api") +@limiter.limit("5 per minute") +def my_api(): + return "API response" + + +if __name__ == "__main__": + app.run() diff --git a/security/jwt_auth.py b/security/jwt_auth.py new file mode 100644 index 0000000..4e7e090 --- /dev/null +++ b/security/jwt_auth.py @@ -0,0 +1,31 @@ +import jwt +import datetime +from dotenv import load_dotenv +import os + +load_dotenv() + +SECRET_KEY = os.getenv('JWT_SECRET_KEY') + +def generate_jwt(user_id): + payload = { + 'user_id': user_id, + 'exp': datetime.datetime.utcnow() + datetime.timedelta(hours=1) + } + user_token = jwt.encode(payload, SECRET_KEY, algorithm='HS256') + return user_token + +def verify_jwt(user_token): + try: + payload = jwt.decode(user_token, SECRET_KEY, algorithms=['HS256']) + return payload['user_id'] + except jwt.ExpiredSignatureError: + return None + except jwt.InvalidTokenError: + return None + +if __name__ == "__main__": + token = generate_jwt('user123') + print(f"Generated JWT: {token}") + user_id = verify_jwt(token) + print(f"Verified User ID: {user_id}") diff --git a/security/otp.py b/security/otp.py new file mode 100644 index 0000000..70caefa --- /dev/null +++ b/security/otp.py @@ -0,0 +1,15 @@ +import pyotp + +# Generate a new secret key +totp = pyotp.TOTP(pyotp.random_base32()) +print("Secret:", totp.secret) + +# Generate a time-based OTP +print("Your OTP:", totp.now()) + +# Verify OTP +otp = input("Enter OTP: ") +if totp.verify(otp): + print("OTP is valid!") +else: + print("Invalid OTP.") diff --git a/security/passkeys.py b/security/passkeys.py new file mode 100644 index 0000000..d2d4ab1 --- /dev/null +++ b/security/passkeys.py @@ -0,0 +1,42 @@ +from flask import Flask, request, jsonify, session +from webauthn import generate_registration_options, verify_registration_response +import os + +app = Flask(__name__) +app.secret_key = os.urandom(24) + +users = {} + + +@app.route('/register', methods=['POST']) +def register(): + email = request.json.get('email') + + registration_options = generate_registration_options( + rp_id="localhost", + rp_name="MyApp", + user_id=email, + user_name=email + ) + + session['registration_options'] = registration_options + return jsonify(registration_options) + + +@app.route('/register/verify', methods=['POST']) +def verify_registration(): + email = request.json.get('email') + response = request.json.get('response') + + registration_options = session.get('registration_options') + + try: + verified = verify_registration_response(response, registration_options) + users[email] = verified + return jsonify({"status": "ok"}), 200 + except Exception as e: + return jsonify({"status": "error", "message": str(e)}), 400 + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/security/rbac_abac.py b/security/rbac_abac.py new file mode 100644 index 0000000..aba0d3a --- /dev/null +++ b/security/rbac_abac.py @@ -0,0 +1,17 @@ +from flask import Flask +from flask_principal import Principal, Permission, RoleNeed + +app = Flask(__name__) +principals = Principal(app) + +admin_permission = Permission(RoleNeed('admin')) + + +@app.route('/admin') +@admin_permission.require(http_exception=403) +def admin_page(): + return "Welcome Admin!" + + +if __name__ == "__main__": + app.run() diff --git a/security/tls_nginx_security.conf b/security/tls_nginx_security.conf new file mode 100644 index 0000000..72f96c7 --- /dev/null +++ b/security/tls_nginx_security.conf @@ -0,0 +1,11 @@ +server { + listen 443 ssl; + server_name yourdomain.com; + + ssl_certificate /etc/letsencrypt/live/yourdomain.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/yourdomain.com/privkey.pem; + + location / { + proxy_pass http://localhost:8000; + } +} diff --git a/sms/README.md b/sms/README.md new file mode 100644 index 0000000..93681a7 --- /dev/null +++ b/sms/README.md @@ -0,0 +1,109 @@ +# SMS + +This folder contains Python scripts for integrating various SMS service providers to send text messages. Each script provides functions for sending SMS using different API clients. + +## Contents + +- [Africa's Talking](#africas-talking) +- [AWS SNS](#aws-sns) +- [Azure SMS](#azure-sms) +- [Nexmo (Vonage)](#nexmo-vonage) +- [Twilio](#twilio) + +--- + +### Africa's Talking + +**Africa's Talking** is a communication API platform providing SMS, voice, and USSD services primarily across Africa. + +- **File**: `africastalking_sms.py` +- **Setup**: + 1. Install `africastalking` library: `pip install africastalking`. + 2. Configure API key and username. + +- **Example Usage**: + ```python + from africastalking_sms import send_sms + send_sms("recipient_number", "Your message here") + ``` + +--- + +### AWS SNS + +**AWS SNS (Simple Notification Service)** allows for sending SMS messages to users worldwide. + +- **File**: `aws_sms.py` +- **Setup**: + 1. Install AWS SDK: `pip install boto3`. + 2. Configure AWS credentials with access to SNS. + +- **Example Usage**: + ```python + from aws_sms import send_sms + send_sms("recipient_number", "Your message here") + ``` + +--- + +### Azure SMS + +**Azure Communication Services** enables SMS messaging through the Azure cloud infrastructure. + +- **File**: `azure_sms.py` +- **Setup**: + 1. Install Azure SDK: `pip install azure-communication-sms`. + 2. Configure Azure Communication Services connection string. + +- **Example Usage**: + ```python + from azure_sms import send_sms + send_sms("recipient_number", "Your message here") + ``` + +--- + +### Nexmo (Vonage) + +**Nexmo** (now Vonage) provides APIs for SMS, voice, and messaging services. + +- **File**: `nexmo_sms.py` +- **Setup**: + 1. Install `vonage` library: `pip install vonage`. + 2. Configure API key and secret. + +- **Example Usage**: + ```python + from nexmo_sms import send_sms + send_sms("recipient_number", "Your message here") + ``` + +--- + +### Twilio + +**Twilio** is a popular cloud communications platform, allowing you to send SMS, voice, and other messaging services. + +- **File**: `twilio_sms.py` +- **Setup**: + 1. Install Twilio SDK: `pip install twilio`. + 2. Configure Twilio account SID, auth token, and messaging service SID or sender number. + +- **Example Usage**: + ```python + from twilio_sms import send_sms + send_sms("recipient_number", "Your message here") + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `sms` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your credentials for each SMS provider. +4. Run the scripts to send SMS messages using the specified provider. + +## Contributing + +Contributions are welcome! If you’d like to add more SMS providers or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/sms/africastalking_sms.py b/sms/africastalking_sms.py new file mode 100644 index 0000000..6d63ef2 --- /dev/null +++ b/sms/africastalking_sms.py @@ -0,0 +1,37 @@ +import os + +import africastalking +from dotenv import load_dotenv + +load_dotenv() + +AT_API_KEY = os.getenv('AT_API_KEY') +AT_USERNAME = os.getenv('AT_USERNAME') + +africastalking.initialize(AT_USERNAME, AT_API_KEY) +sms = africastalking.SMS + + +def send_africastalking_sms(to_phone_number, message_body): + """ + Sends an SMS message using Africa's Talking service. + + Args: + to_phone_number (str): Recipient's phone number. + message_body (str): The SMS message to be sent. + + Returns: + dict: Response from Africa's Talking with status and message details. + """ + sms_response = None + try: + sms_response = sms.send(message_body, [to_phone_number]) + print(f"Message sent successfully: {sms_response}") + except Exception as e: + print(f"Failed to send message: {str(e)}") + return sms_response + + +if __name__ == "__main__": + response = send_africastalking_sms('+XXXXXXXXXXX', 'Hello from Africa\'s Talking!') + print(response) diff --git a/sms/aws_sms.py b/sms/aws_sms.py new file mode 100644 index 0000000..50a874a --- /dev/null +++ b/sms/aws_sms.py @@ -0,0 +1,44 @@ +import os + +import boto3 +from dotenv import load_dotenv + +load_dotenv() + +AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') +AWS_REGION = os.getenv('AWS_REGION') + + +def send_aws_sns_sms(to_phone_number, message_body): + """ + Sends an SMS message using AWS SNS. + + Args: + to_phone_number (str): Recipient's phone number. + message_body (str): The SMS message to be sent. + + Returns: + dict: Response from AWS SNS with status and message details. + """ + sms_response = None + sns_client = boto3.client( + "sns", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION + ) + + try: + sms_response = sns_client.publish( + PhoneNumber=to_phone_number, + Message=message_body + ) + print(f"Message sent successfully: {sms_response}") + except Exception as e: + print(f"Failed to send message: {str(e)}") + + return sms_response + +if __name__ == "__main__": + send_aws_sns_sms('+XXXXXXXXXX', 'Hello from AWS SNS!') diff --git a/sms/azure_sms.py b/sms/azure_sms.py new file mode 100644 index 0000000..3b281d1 --- /dev/null +++ b/sms/azure_sms.py @@ -0,0 +1,38 @@ +import os + +from azure.communication.sms import SmsClient +from dotenv import load_dotenv + +load_dotenv() + +AZURE_COMMUNICATION_CONNECTION_STRING = os.getenv('AZURE_COMMUNICATION_CONNECTION_STRING') + + +def send_azure_sms(to_phone_number, message_body): + """ + Sends an SMS message using Azure Communication Services. + + Args: + to_phone_number (str): Recipient's phone number. + message_body (str): The SMS message to be sent. + + Returns: + str: The response from Azure Communication Services. + """ + sms_client = SmsClient.from_connection_string(AZURE_COMMUNICATION_CONNECTION_STRING) + + try: + response = sms_client.send( + from_='', + to=[to_phone_number], + message=message_body, + enable_delivery_report=True + ) + print("Message sent successfully:", response) + except Exception as e: + print(f"Failed to send message: {str(e)}") + + +# Example usage +if __name__ == "__main__": + send_azure_sms('+XXXXXXXXXXX', 'Hello from Azure Communication Services!') diff --git a/sms/nexmo_sms.py b/sms/nexmo_sms.py new file mode 100644 index 0000000..96b1872 --- /dev/null +++ b/sms/nexmo_sms.py @@ -0,0 +1,42 @@ +import os + +import vonage +from dotenv import load_dotenv + +load_dotenv() + +NEXMO_API_KEY = os.getenv('NEXMO_API_KEY') +NEXMO_API_SECRET = os.getenv('NEXMO_API_SECRET') + +client = vonage.Client(key=NEXMO_API_KEY, secret=NEXMO_API_SECRET) +sms = vonage.Sms(client) + + +def send_nexmo_sms(to_phone_number, message_body): + """ + Sends an SMS message using Nexmo (Vonage) service. + + Args: + to_phone_number (str): Recipient's phone number. + message_body (str): The SMS message to be sent. + + Returns: + dict: Response from Nexmo with status and message details. + """ + response_data = sms.send_message({ + 'from': 'NEXMO', + 'to': to_phone_number, + 'text': message_body + }) + + if response_data["messages"][0]["status"] == "0": + print("Message sent successfully.") + else: + print(f"Message failed with error: {response_data['messages'][0]['error-text']}") + + return response_data + + +if __name__ == "__main__": + response = send_nexmo_sms('+XXXXXXXXXX', 'Hey, Steve trying to reach you!') + print(response) diff --git a/sms/twilio_sms.py b/sms/twilio_sms.py new file mode 100644 index 0000000..bcbb139 --- /dev/null +++ b/sms/twilio_sms.py @@ -0,0 +1,36 @@ +import os + +from dotenv import load_dotenv +from twilio.rest import Client + +load_dotenv() + +TWILIO_ACCOUNT_SID = os.getenv('TWILIO_ACCOUNT_SID') +TWILIO_AUTH_TOKEN = os.getenv('TWILIO_AUTH_TOKEN') +TWILIO_PHONE_NUMBER = os.getenv('TWILIO_PHONE_NUMBER') + + +def send_twilio_sms(to_phone_number, message_body): + """ + Sends an SMS message using the Twilio service. + + Args: + to_phone_number (str): Recipient's phone number. + message_body (str): The SMS message to be sent. + + Returns: + str: Message SID if the message was sent successfully. + """ + client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN) + + message = client.messages.create( + body=message_body, + from_=TWILIO_PHONE_NUMBER, + to=to_phone_number + ) + return message.sid + + +if __name__ == "__main__": + sid = send_twilio_sms('+XXXXXXXXXX', 'Hey, Steve trying to reach you!') + print(f"Message sent with SID: {sid}") diff --git a/storage/README.md b/storage/README.md new file mode 100644 index 0000000..66e4b7b --- /dev/null +++ b/storage/README.md @@ -0,0 +1,95 @@ +# Storage + +This folder contains Python scripts for integrating various cloud storage services, providing functions for uploading, downloading, and managing files across different providers. + +## Contents + +- [AWS S3 Storage](#aws-s3-storage) +- [Azure Blob Storage](#azure-blob-storage) +- [Google Cloud Storage](#google-cloud-storage) +- [Google Drive Storage](#google-drive-storage) + +--- + +### AWS S3 Storage + +**Amazon S3** is a scalable storage service by Amazon Web Services, commonly used for file storage and backups. + +- **File**: `aws_s3_storage.py` +- **Setup**: + 1. Install AWS SDK: `pip install boto3`. + 2. Configure AWS credentials with access to S3. + +- **Example Usage**: + ```python + from aws_s3_storage import upload_file, download_file + upload_file("local_file.txt", "my-s3-bucket", "s3_file.txt") + download_file("my-s3-bucket", "s3_file.txt", "downloaded_file.txt") + ``` + +--- + +### Azure Blob Storage + +**Azure Blob Storage** is Microsoft's cloud storage solution for storing large amounts of unstructured data. + +- **File**: `azure_blob_storage.py` +- **Setup**: + 1. Install Azure SDK: `pip install azure-storage-blob`. + 2. Configure Azure storage account and access keys. + +- **Example Usage**: + ```python + from azure_blob_storage import upload_blob, download_blob + upload_blob("local_file.txt", "my-container", "blob_name.txt") + download_blob("my-container", "blob_name.txt", "downloaded_file.txt") + ``` + +--- + +### Google Cloud Storage + +**Google Cloud Storage** is a unified object storage service by Google, suitable for storing large datasets and backups. + +- **File**: `gcp_storage.py` +- **Setup**: + 1. Install Google Cloud client: `pip install google-cloud-storage`. + 2. Configure Google Cloud credentials. + +- **Example Usage**: + ```python + from gcp_storage import upload_file, download_file + upload_file("local_file.txt", "my-gcs-bucket", "gcs_file.txt") + download_file("my-gcs-bucket", "gcs_file.txt", "downloaded_file.txt") + ``` + +--- + +### Google Drive Storage + +**Google Drive** provides file storage and synchronization service by Google, allowing users to store and share files. + +- **File**: `google_drive_storage.py` +- **Setup**: + 1. Install Google Drive client: `pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client`. + 2. Configure OAuth credentials for Google Drive API. + +- **Example Usage**: + ```python + from google_drive_storage import upload_to_drive, download_from_drive + upload_to_drive("local_file.txt", "Drive Folder ID") + download_from_drive("Drive File ID", "downloaded_file.txt") + ``` + +--- + +## How to Use + +1. Clone the repository and navigate to the `storage` folder. +2. Install any required dependencies as mentioned in each section. +3. Configure your credentials for each storage provider. +4. Run the scripts to upload or download files using the specified storage service. + +## Contributing + +Contributions are welcome! If you’d like to add more storage providers or enhance existing implementations, please submit a pull request with a detailed description of your changes. diff --git a/storage/aws_s3_storage.py b/storage/aws_s3_storage.py new file mode 100644 index 0000000..94eb6b1 --- /dev/null +++ b/storage/aws_s3_storage.py @@ -0,0 +1,77 @@ +import os +import re + +import boto3 +from dotenv import load_dotenv + +load_dotenv() + +AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') +AWS_REGION = os.getenv('AWS_REGION') +AWS_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME') + +s3 = boto3.client( + 's3', + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY +) + + +def upload_file_to_s3(file_path, object_name): + """ + Uploads a file to S3. + + Args: + file_path (str): Path to the file to upload. + object_name (str): S3 object name. + """ + try: + s3.upload_file(file_path, AWS_BUCKET_NAME, object_name) + print(f"File uploaded successfully to {object_name}") + except Exception as e: + print(f"Error uploading file: {e}") + + +def download_file_from_s3(object_name, file_path): + """ + Downloads a file from S3. + + Args: + object_name (str): S3 object name. + file_path (str): Local path to save the file. + """ + try: + s3.download_file(AWS_BUCKET_NAME, object_name, file_path) + print(f"File downloaded successfully to {file_path}") + except Exception as e: + print(f"Error downloading file: {e}") + + +def delete_file_from_s3(file_url): + """ + Deletes a file from S3 using the provided S3 URL. + + Args: + file_url (str): The full S3 URL of the file to delete. + """ + try: + match = re.match(r"https://(.*)\.s3\.amazonaws\.com/(.*)", file_url) + if match: + bucket_name = match.group(1) + object_key = match.group(2) + else: + raise ValueError("Invalid S3 URL format") + s3.delete_object(Bucket=bucket_name, Key=object_key) + print(f"File {object_key} from bucket {bucket_name} deleted successfully") + + except Exception as e: + print(f"Error deleting file: {e}") + + +# Example usage +if __name__ == "__main__": + upload_file_to_s3("/path/to/file.txt", "uploaded_file.txt") + download_file_from_s3("uploaded_file.txt", "/path/to/downloaded_file.txt") + delete_file_from_s3("https://your-bucket.s3.amazonaws.com/uploaded_file.txt") diff --git a/storage/azure_blob_storage.py b/storage/azure_blob_storage.py new file mode 100644 index 0000000..a3710a3 --- /dev/null +++ b/storage/azure_blob_storage.py @@ -0,0 +1,75 @@ +import os +from urllib.parse import urlparse + +from azure.storage.blob import BlobServiceClient +from dotenv import load_dotenv + +load_dotenv() + +AZURE_CONNECTION_STRING = os.getenv('AZURE_CONNECTION_STRING') +AZURE_CONTAINER_NAME = os.getenv('AZURE_CONTAINER_NAME') + +blob_service_client = BlobServiceClient.from_connection_string(AZURE_CONNECTION_STRING) + + +# Upload a file +def upload_file_to_azure(file_path, blob_name): + """ + Uploads a file to Azure Blob Storage. + + Args: + file_path (str): Path to the file to upload. + blob_name (str): Name of the blob (object). + """ + blob_client = blob_service_client.get_blob_client(container=AZURE_CONTAINER_NAME, blob=blob_name) + + try: + with open(file_path, "rb") as data: + blob_client.upload_blob(data) + print(f"File {file_path} uploaded successfully to {blob_name}") + except Exception as e: + print(f"Error uploading file: {e}") + + +# Download a file +def download_file_from_azure(blob_name, file_path): + """ + Downloads a file from Azure Blob Storage. + + Args: + blob_name (str): Name of the blob (object). + file_path (str): Path to save the downloaded file. + """ + blob_client = blob_service_client.get_blob_client(container=AZURE_CONTAINER_NAME, blob=blob_name) + + try: + with open(file_path, "wb") as file: + file.write(blob_client.download_blob().readall()) + print(f"File {blob_name} downloaded successfully to {file_path}") + except Exception as e: + print(f"Error downloading file: {e}") + + +def delete_file_from_azure(blob_url): + """ + Deletes a file from Azure Blob Storage using the blob's URL. + + Args: + blob_url (str): URL of the blob to delete. + """ + try: + parsed_url = urlparse(blob_url) + blob_name = parsed_url.path.lstrip(f'/{AZURE_CONTAINER_NAME}/') + + blob_client = blob_service_client.get_blob_client(container=AZURE_CONTAINER_NAME, blob=blob_name) + + blob_client.delete_blob() + print(f"Blob {blob_name} deleted successfully") + except Exception as e: + print(f"Error deleting blob: {e}") + + +if __name__ == "__main__": + upload_file_to_azure("/path/to/file.txt", "folder1/folder2/uploaded_file.txt") + download_file_from_azure("folder1/folder2/uploaded_file.txt", "/path/to/downloaded_file.txt") + delete_file_from_azure("https://youraccount.blob.core.windows.net/yourcontainer/folder1/folder2/uploaded_file.txt") diff --git a/storage/gcp_storage.py b/storage/gcp_storage.py new file mode 100644 index 0000000..889142a --- /dev/null +++ b/storage/gcp_storage.py @@ -0,0 +1,81 @@ +import os +from urllib.parse import urlparse + +from dotenv import load_dotenv +from google.cloud import storage + +load_dotenv() + +GOOGLE_CLOUD_PROJECT = os.getenv('GOOGLE_CLOUD_PROJECT') +GCP_BUCKET_NAME = os.getenv('GCP_BUCKET_NAME') + + +def get_gcp_client(): + return storage.Client() + + +def upload_file_to_gcs(file_path, blob_name): + """ + Uploads a file to Google Cloud Storage. + + Args: + file_path (str): Path to the file to upload. + blob_name (str): Name of the blob (object). + """ + client = get_gcp_client() + bucket = client.bucket(GCP_BUCKET_NAME) + blob = bucket.blob(blob_name) + + try: + blob.upload_from_filename(file_path) + print(f"File {file_path} uploaded successfully to {blob_name}") + except Exception as e: + print(f"Error uploading file: {e}") + + +def download_file_from_gcs(blob_name, file_path): + """ + Downloads a file from Google Cloud Storage. + + Args: + blob_name (str): Name of the blob (object). + file_path (str): Path to save the downloaded file. + """ + client = get_gcp_client() + bucket = client.bucket(GCP_BUCKET_NAME) + blob = bucket.blob(blob_name) + + try: + blob.download_to_filename(file_path) + print(f"File {blob_name} downloaded successfully to {file_path}") + except Exception as e: + print(f"Error downloading file: {e}") + + +def delete_file_from_gcs(file_url): + """ + Deletes a file from Google Cloud Storage given a full URL. + + Args: + file_url (str): The full URL of the file in GCS. + """ + client = get_gcp_client() + + try: + parsed_url = urlparse(file_url) + bucket_name = parsed_url.netloc + blob_name = parsed_url.path.lstrip('/') + + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + blob.delete() + print(f"File {blob_name} deleted successfully from bucket {bucket_name}") + except Exception as e: + print(f"Error deleting file: {e}") + + +if __name__ == "__main__": + upload_file_to_gcs("/path/to/file.txt", "uploaded_file.txt") + download_file_from_gcs("uploaded_file.txt", "/path/to/downloaded_file.txt") + delete_file_from_gcs("https://storage.googleapis.com/your-bucket/uploaded_file.txt") diff --git a/storage/google_drive_storage.py b/storage/google_drive_storage.py new file mode 100644 index 0000000..4daf827 --- /dev/null +++ b/storage/google_drive_storage.py @@ -0,0 +1,88 @@ +import os +from io import BytesIO + +from dotenv import load_dotenv +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload + +load_dotenv() + +SERVICE_ACCOUNT_FILE = os.getenv('GOOGLE_DRIVE_SERVICE_ACCOUNT_FILE') + +SCOPES = ['https://www.googleapis.com/auth/drive'] + +credentials = service_account.Credentials.from_service_account_file( + SERVICE_ACCOUNT_FILE, scopes=SCOPES) + +drive_service = build('drive', 'v3', credentials=credentials) + + +def upload_file_to_drive(file_path, file_name): + """ + Upload a file to Google Drive. + + Args: + file_path (str): Path to the local file. + file_name (str): Name of the file on Google Drive. + + Returns: + dict: File metadata response from Google Drive. + """ + file_metadata = {'name': file_name} + media = MediaFileUpload(file_path, resumable=True) + + try: + file = drive_service.files().create(body=file_metadata, media_body=media, fields='id').execute() + print(f"File uploaded successfully. File ID: {file.get('id')}") + return file + except Exception as e: + print(f"Error uploading file: {e}") + return None + + +def download_file_from_drive(g_file_id, destination): + """ + Download a file from Google Drive. + + Args: + g_file_id (str): ID of the file on Google Drive. + destination (str): Local path to save the downloaded file. + """ + try: + request = drive_service.files().get_media(fileId=g_file_id) + fh = BytesIO() + downloader = MediaIoBaseDownload(fh, request) + + done = False + while not done: + status, done = downloader.next_chunk() + print(f"Download {int(status.progress() * 100)}%.") + + with open(destination, 'wb') as f: + f.write(fh.getvalue()) + print(f"File downloaded successfully to {destination}") + except Exception as e: + print(f"Error downloading file: {e}") + + +def delete_file_from_drive(g_file_id): + """ + Delete a file from Google Drive. + + Args: + g_file_id (str): ID of the file on Google Drive. + """ + try: + drive_service.files().delete(fileId=g_file_id).execute() + print(f"File with ID {g_file_id} deleted successfully.") + except Exception as e: + print(f"Error deleting file: {e}") + + +if __name__ == "__main__": + uploaded_file = upload_file_to_drive("/path/to/file.txt", "uploaded_file.txt") + if uploaded_file: + file_id = uploaded_file.get('id') + download_file_from_drive(file_id, "/path/to/downloaded_file.txt") + delete_file_from_drive(file_id)