From b1085f41f3cb5f334430dddca70b5817a8ed8756 Mon Sep 17 00:00:00 2001 From: Louis Faucon Date: Tue, 22 Aug 2023 22:18:24 +0200 Subject: [PATCH 1/3] refactor: simplify secret fecth script --- infra/ansible/scripts/get-vm-secrets.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/infra/ansible/scripts/get-vm-secrets.sh b/infra/ansible/scripts/get-vm-secrets.sh index 287db999d0..c99eaf9db7 100755 --- a/infra/ansible/scripts/get-vm-secrets.sh +++ b/infra/ansible/scripts/get-vm-secrets.sh @@ -3,11 +3,13 @@ export VM_ADDR="${1:-"staging.tournesol.app"}" export VM_USER="${2:-"$USER"}" -function get_settings_value() { - local jq_filter=$1; +SETTINGS_FILE_JSON="$(\ ssh "$VM_USER@$VM_ADDR" -- \ - 'sudo python3 -c '\''import yaml,json; print(json.dumps(yaml.safe_load(open("/etc/tournesol/settings.yaml"))))'\'' \ - | jq -r' "'$jq_filter | values'" + 'sudo python3 -c '\''import yaml,json; print(json.dumps(yaml.safe_load(open("/etc/tournesol/settings.yaml"))))'\'''\ +)" + +function get_settings_value() { + jq -r $1 <<< "$SETTINGS_FILE_JSON" } DJANGO_SECRET_KEY=$(get_settings_value .SECRET_KEY) From b9569285658271bc44e7dd96f3053b1f64ee71ef Mon Sep 17 00:00:00 2001 From: Louis Faucon Date: Tue, 22 Aug 2023 22:23:46 +0200 Subject: [PATCH 2/3] feat: Adding script and service definition for discord link commenting bot --- .../roles/discordbot/files/discord_bot.py | 31 +++++++++++++++++++ .../roles/discordbot/handlers/main.yaml | 5 +++ .../ansible/roles/discordbot/tasks/main.yaml | 30 ++++++++++++++++++ .../templates/discordbot.service.j2 | 11 +++++++ .../roles/django/templates/settings.yaml.j2 | 2 ++ .../ansible/scripts/deploy-without-secrets.sh | 5 +++ infra/ansible/scripts/forget-secrets.sh | 1 + infra/ansible/scripts/get-vm-secrets.sh | 3 ++ infra/ansible/setup-discord.yml | 4 +++ 9 files changed, 92 insertions(+) create mode 100644 infra/ansible/roles/discordbot/files/discord_bot.py create mode 100644 infra/ansible/roles/discordbot/handlers/main.yaml create mode 100644 infra/ansible/roles/discordbot/tasks/main.yaml create mode 100644 infra/ansible/roles/discordbot/templates/discordbot.service.j2 create mode 100644 infra/ansible/setup-discord.yml diff --git a/infra/ansible/roles/discordbot/files/discord_bot.py b/infra/ansible/roles/discordbot/files/discord_bot.py new file mode 100644 index 0000000000..ce6e3fa784 --- /dev/null +++ b/infra/ansible/roles/discordbot/files/discord_bot.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import discord +import re +import os + +intents = discord.Intents.default() +intents.message_content = True + +client = discord.Client(intents=intents) + +yt_url_re = re.compile(r"\b(?:https?:)?(?:\/\/)?(?:youtu\.be\/|(?:www\.|m\.)?youtube\.com\/(?:watch|v|embed|live)(?:\.php)?(?:\?\S*v=|\/))([a-zA-Z0-9\_-]{11})(?:[\?&][a-zA-Z0-9\_-]+=[a-zA-Z0-9\_-]+)*(?:[&\/\#].*)?\b") + +@client.event +async def on_ready(): + print(f'We have logged in as {client.user}') + +@client.event +async def on_message(message): + if message.author == client.user: + return + url_matches = set(yt_url_re.findall(message.content)) + if len(url_matches) > 0: + print(f'Found a message containing video ids: {", ".join(url_matches)}') + links = [f"https://tournesol.app/entities/yt:{video_id}" for video_id in url_matches] + msg = f"Please use Tournesol links when posting on social media to help us promote the project:\n" + "\n".join(links) + if message.content: + await message.channel.send(msg) + +token = os.environ["DISCORD_BOT_TOKEN"] +client.run(token) diff --git a/infra/ansible/roles/discordbot/handlers/main.yaml b/infra/ansible/roles/discordbot/handlers/main.yaml new file mode 100644 index 0000000000..0605afb94a --- /dev/null +++ b/infra/ansible/roles/discordbot/handlers/main.yaml @@ -0,0 +1,5 @@ +- name: Restart Discord Bot + systemd: + name: discordbot.service + state: restarted + daemon_reload: true \ No newline at end of file diff --git a/infra/ansible/roles/discordbot/tasks/main.yaml b/infra/ansible/roles/discordbot/tasks/main.yaml new file mode 100644 index 0000000000..b028d7b837 --- /dev/null +++ b/infra/ansible/roles/discordbot/tasks/main.yaml @@ -0,0 +1,30 @@ +- name: Create Virtualenv for Discord Bot + pip: + name: + - "discord.py==2.3.2" + - "pip==23.2.1" + virtualenv: /srv/tournesol-discordbot/venv + virtualenv_python: python3.9 + become: yes + +- name: Copy Discord Bot service file + template: + src: discordbot.service.j2 + dest: /etc/systemd/system/discordbot.service + notify: Restart Discord Bot + +- name: Copy Discord Bot script file + copy: + src: discord_bot.py + dest: /usr/local/bin/discord_bot.py + mode: u=rx,o=,g= + notify: Restart Discord Bot + +- name: Enable and start Discord Bot + systemd: + name: discordbot.service + enabled: true + state: started + daemon_reload: true + +- meta: flush_handlers diff --git a/infra/ansible/roles/discordbot/templates/discordbot.service.j2 b/infra/ansible/roles/discordbot/templates/discordbot.service.j2 new file mode 100644 index 0000000000..291be9ca24 --- /dev/null +++ b/infra/ansible/roles/discordbot/templates/discordbot.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Discord bot service +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/bash -c "source /srv/tournesol-discordbot/venv/bin/activate && /usr/local/bin/discord_bot.py" +Environment="DISCORD_BOT_TOKEN={{discord_bot_token}}" + +[Install] +WantedBy=multi-user.target diff --git a/infra/ansible/roles/django/templates/settings.yaml.j2 b/infra/ansible/roles/django/templates/settings.yaml.j2 index 7e3a229363..227ef0671f 100644 --- a/infra/ansible/roles/django/templates/settings.yaml.j2 +++ b/infra/ansible/roles/django/templates/settings.yaml.j2 @@ -77,6 +77,8 @@ TWITTERBOT_CREDENTIALS: "ACCESS_TOKEN_SECRET": "{{access_token_secret_twitterbot_en}}", } +DISCORD_BOT_TOKEN: {{discord_bot_token}} + DISCORD_CHANNEL_WEBHOOKS: { infra_alert: "{{ discord_infra_alert_webhook }}", infra_alert_private: "{{ discord_infra_alert_private_webhook }}", diff --git a/infra/ansible/scripts/deploy-without-secrets.sh b/infra/ansible/scripts/deploy-without-secrets.sh index f73053eb9a..5f56c48f6d 100755 --- a/infra/ansible/scripts/deploy-without-secrets.sh +++ b/infra/ansible/scripts/deploy-without-secrets.sh @@ -17,6 +17,10 @@ if [[ "${2:-""}" == "fast" ]] then echo "Using setup-fast.yaml" SETUP_FILE="setup-fast.yml" +elif [[ "${2:-""}" == "discord" ]] +then + echo "Using setup-discord.yaml" + SETUP_FILE="setup-discord.yml" else echo "Using setup.yaml" SETUP_FILE="setup.yml" @@ -53,6 +57,7 @@ ansible-playbook -i inventory.yml -l "$ANSIBLE_HOST" "$SETUP_FILE" \ -e "consumer_secret_twitterbot_en=${TWBOT_CONSUMER_SECRET_EN:-""}" \ -e "access_token_twitterbot_en=${TWBOT_ACCESS_TOKEN_EN:-""}" \ -e "access_token_secret_twitterbot_en=${TWBOT_ACCESS_TOKEN_SECRET_EN:-""}" \ + -e "discord_bot_token=${DISCORD_BOT_TOKEN:-""}" \ -e "discord_infra_alert_webhook=${DISCORD_INFRA_ALERT_WEBHOOK:-""}" \ -e "discord_infra_alert_private_webhook=${DISCORD_INFRA_ALERT_PRIVATE_WEBHOOK:-""}" \ -e "discord_twitter_webhook=${DISCORD_TWITTER_WEBHOOK:-""}" \ diff --git a/infra/ansible/scripts/forget-secrets.sh b/infra/ansible/scripts/forget-secrets.sh index 8cbc209668..a3a355c1f9 100755 --- a/infra/ansible/scripts/forget-secrets.sh +++ b/infra/ansible/scripts/forget-secrets.sh @@ -26,6 +26,7 @@ unset TWBOT_CONSUMER_KEY_EN unset TWBOT_CONSUMER_SECRET_EN unset TWBOT_ACCESS_TOKEN_EN unset TWBOT_ACCESS_TOKEN_SECRET_EN +unset DISCORD_BOT_TOKEN unset DISCORD_INFRA_ALERT_WEBHOOK unset DISCORD_INFRA_ALERT_PRIVATE_WEBHOOK unset DISCORD_TWITTER_WEBHOOK diff --git a/infra/ansible/scripts/get-vm-secrets.sh b/infra/ansible/scripts/get-vm-secrets.sh index c99eaf9db7..d7252626d9 100755 --- a/infra/ansible/scripts/get-vm-secrets.sh +++ b/infra/ansible/scripts/get-vm-secrets.sh @@ -90,6 +90,9 @@ export TWBOT_ACCESS_TOKEN_EN TWBOT_ACCESS_TOKEN_SECRET_EN=$(get_settings_value .TWITTERBOT_CREDENTIALS.\"@TournesolBot\".ACCESS_TOKEN_SECRET) export TWBOT_ACCESS_TOKEN_SECRET_EN +DISCORD_BOT_TOKEN=$(get_settings_value .DISCORD_BOT_TOKEN) +export DISCORD_BOT_TOKEN + DISCORD_INFRA_ALERT_WEBHOOK=$(get_settings_value .DISCORD_CHANNEL_WEBHOOKS.infra_alert) export DISCORD_INFRA_ALERT_WEBHOOK diff --git a/infra/ansible/setup-discord.yml b/infra/ansible/setup-discord.yml new file mode 100644 index 0000000000..a09eea1425 --- /dev/null +++ b/infra/ansible/setup-discord.yml @@ -0,0 +1,4 @@ +- hosts: tournesol + roles: + - discordbot + become: true From 849e6a3f84827b50a1d191cd485492cdd69ef51f Mon Sep 17 00:00:00 2001 From: jst Date: Sun, 27 Aug 2023 21:16:55 +0200 Subject: [PATCH 3/3] do not reply to videos IDs that do not exist in Tournesol --- .../roles/discordbot/files/discord_bot.py | 24 +++++++++++++++---- .../ansible/roles/discordbot/tasks/main.yaml | 1 + .../templates/discordbot.service.j2 | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/infra/ansible/roles/discordbot/files/discord_bot.py b/infra/ansible/roles/discordbot/files/discord_bot.py index ce6e3fa784..dca1974364 100644 --- a/infra/ansible/roles/discordbot/files/discord_bot.py +++ b/infra/ansible/roles/discordbot/files/discord_bot.py @@ -3,6 +3,7 @@ import discord import re import os +import requests intents = discord.Intents.default() intents.message_content = True @@ -11,6 +12,20 @@ yt_url_re = re.compile(r"\b(?:https?:)?(?:\/\/)?(?:youtu\.be\/|(?:www\.|m\.)?youtube\.com\/(?:watch|v|embed|live)(?:\.php)?(?:\?\S*v=|\/))([a-zA-Z0-9\_-]{11})(?:[\?&][a-zA-Z0-9\_-]+=[a-zA-Z0-9\_-]+)*(?:[&\/\#].*)?\b") + +token = os.environ["DISCORD_BOT_TOKEN"] +tournesol_api_url = os.environ["TOURNESOL_API_URL"] + + +def video_exists(video_id: str) -> bool: + resp = requests.head(f"{tournesol_api_url}/video/{video_id}/") + if resp.status_code != requests.codes.ok: + print(f"video {video_id} not found in Tournesol") + return False + print(f"video {video_id} found in Tournesol") + return True + + @client.event async def on_ready(): print(f'We have logged in as {client.user}') @@ -22,10 +37,11 @@ async def on_message(message): url_matches = set(yt_url_re.findall(message.content)) if len(url_matches) > 0: print(f'Found a message containing video ids: {", ".join(url_matches)}') - links = [f"https://tournesol.app/entities/yt:{video_id}" for video_id in url_matches] - msg = f"Please use Tournesol links when posting on social media to help us promote the project:\n" + "\n".join(links) - if message.content: + links = [f"https://tournesol.app/entities/yt:{video_id}" for video_id in url_matches if video_exists(video_id)] + if len(links) > 0: + msg = f"Please use Tournesol links when posting on social media to help us promote the project:\n" + "\n".join(links) + # if message.content: await message.channel.send(msg) -token = os.environ["DISCORD_BOT_TOKEN"] + client.run(token) diff --git a/infra/ansible/roles/discordbot/tasks/main.yaml b/infra/ansible/roles/discordbot/tasks/main.yaml index b028d7b837..30aa3bff86 100644 --- a/infra/ansible/roles/discordbot/tasks/main.yaml +++ b/infra/ansible/roles/discordbot/tasks/main.yaml @@ -3,6 +3,7 @@ name: - "discord.py==2.3.2" - "pip==23.2.1" + - "requests==2.31.0" virtualenv: /srv/tournesol-discordbot/venv virtualenv_python: python3.9 become: yes diff --git a/infra/ansible/roles/discordbot/templates/discordbot.service.j2 b/infra/ansible/roles/discordbot/templates/discordbot.service.j2 index 291be9ca24..cd75d3fd8a 100644 --- a/infra/ansible/roles/discordbot/templates/discordbot.service.j2 +++ b/infra/ansible/roles/discordbot/templates/discordbot.service.j2 @@ -6,6 +6,7 @@ After=network.target Type=simple ExecStart=/usr/bin/bash -c "source /srv/tournesol-discordbot/venv/bin/activate && /usr/local/bin/discord_bot.py" Environment="DISCORD_BOT_TOKEN={{discord_bot_token}}" +Environment="TOURNESOL_API_URL={{api_scheme}}://{{api_domain_name}}" [Install] WantedBy=multi-user.target