diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ced3ae5..29ffda8 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -60,4 +60,5 @@ jobs: DISCORD_CHANNEL_ID: ${{ secrets.DISCORD_CHANNEL_ID }} DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }} DISCORD_TESTER_TOKEN: ${{ secrets.DISCORD_TESTER_TOKEN }} + DISCORD_USER_ID: ${{ secrets.DISCORD_USER_ID }} VOTING_EMOJIS: ${{ secrets.VOTING_EMOJIS }} diff --git a/.gitignore b/.gitignore index f10e125..e38e5a4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ docker/.data test-locally.sh run-locally.sh +.idea +vote_backups diff --git a/src/agenda_cultural/api.rs b/src/agenda_cultural/api.rs index aec2df7..525801b 100644 --- a/src/agenda_cultural/api.rs +++ b/src/agenda_cultural/api.rs @@ -7,6 +7,7 @@ use lazy_static::lazy_static; use reqwest::{Client, Response}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::policies::ExponentialBackoff; +use reqwest_retry::Jitter::Bounded; use reqwest_retry::RetryTransientMiddleware; use scraper::{Html, Selector}; use std::cmp::Ordering; @@ -25,8 +26,9 @@ lazy_static! { static ref REST_CLIENT: ClientWithMiddleware = ClientBuilder::new(Client::new()) .with(RetryTransientMiddleware::new_with_policy( ExponentialBackoff::builder() - .retry_bounds(Duration::from_millis(50), Duration::from_millis(500)) - .build_with_total_retry_duration_and_max_retries(Duration::from_secs(3)) + .jitter(Bounded) + .retry_bounds(Duration::from_millis(50), Duration::from_millis(1000)) + .build_with_total_retry_duration_and_max_retries(Duration::from_secs(10)) )) .build(); static ref EVENT_ID_SELECTOR: Selector = Selector::parse(&format!( diff --git a/src/discord/api.rs b/src/discord/api.rs index b2e1d1f..ed977ba 100644 --- a/src/discord/api.rs +++ b/src/discord/api.rs @@ -9,7 +9,7 @@ use serenity::all::ReactionType::{Custom, Unicode}; use serenity::all::{ AutoArchiveDuration, ChannelType, Colour, CreateEmbedAuthor, CreateThread, CurrentUser, EditThread, Embed, GatewayIntents, GetMessages, GuildChannel, Message, MessageId, - MessageReaction, PartialGuild, PrivateChannel, ReactionType, User, + MessageReaction, PartialGuild, PrivateChannel, ReactionType, User, UserId, }; use serenity::builder::{CreateEmbed, CreateMessage, EditMessage}; use serenity::cache::Settings; @@ -181,7 +181,10 @@ impl DiscordAPI { .await } - #[instrument(skip(self, message), fields(event = %message.embeds.first().map(|embed| embed.url.clone().unwrap()).unwrap_or_default() + #[instrument( + skip(self, message), + fields(event = %message.embeds.first() + .map(|embed| embed.url.clone().unwrap()).unwrap_or_default() ))] pub async fn add_reaction_to_message(&self, message: &Message, emoji_char: char) { let react_result = message @@ -197,7 +200,10 @@ impl DiscordAPI { } } - #[instrument(skip(self, message), fields(event = %message.embeds.first().map(|embed| embed.url.clone().unwrap()).unwrap_or_default() + #[instrument( + skip(self, message), + fields(event = %message.embeds.first() + .map(|embed| embed.url.clone().unwrap()).unwrap_or_default() ))] pub async fn tag_save_for_later_reactions(&self, message: &mut Message, emoji_char: char) { let save_for_later_reaction = ReactionType::from(emoji_char); @@ -269,26 +275,29 @@ impl DiscordAPI { .expect("Failed to edit message!"); } - #[instrument(skip(self, event_message, vote_emojis), fields(event = %event_message.embeds.first().map(|embed| embed.url.clone().unwrap()).unwrap_or_default() + #[instrument(skip_all, + fields(event = %event_message.embeds.first() + .map(|embed| embed.url.clone().unwrap()).unwrap_or_default() ))] pub async fn send_privately_users_review( &self, event_message: &Message, vote_emojis: &[EmojiConfig; 5], - ) { + ) -> Vec { + let mut users_with_reviews = Vec::new(); let mut event_embed = event_message.embeds.first().cloned().unwrap(); let event_url = event_embed.url.clone(); if event_url.is_none() { warn!("Event has no URL!"); - return; + return users_with_reviews; } let users_votes = self.get_user_votes(event_message, vote_emojis).await; if users_votes.is_empty() { trace!("No user has voted on this message"); - return; + return users_with_reviews; } let event_url = event_url.unwrap(); @@ -297,10 +306,15 @@ impl DiscordAPI { for (vote, users) in users_votes.iter().enumerate() { for user in users.iter().filter(|user| !user.bot) { + if !users_with_reviews.contains(&user.id) { + users_with_reviews.push(user.id); + } self.send_user_review(user, &event_url, event_embed.clone(), vote_emojis, vote) .await; } } + + users_with_reviews } async fn send_user_review( @@ -452,17 +466,13 @@ impl DiscordAPI { description: Option, ) -> CreateEmbed { match &comment { - None => CreateEmbed::from(event_embed).description(format!( - "{}\n**Voto:** {}", - description.unwrap(), - vote_emoji - )), - Some(comment) => CreateEmbed::from(event_embed).description(format!( - "{}\n**Voto:** {}\n**Comentários:** {}", - description.unwrap(), - vote_emoji, - comment.content - )), + None => CreateEmbed::from(event_embed) + .description(description.unwrap()) + .field("Voto", vote_emoji.to_string(), true), + Some(comment) => CreateEmbed::from(event_embed) + .description(description.unwrap()) + .field("Voto", vote_emoji.to_string(), true) + .field("Comentários", &comment.content, true), } } diff --git a/src/discord/backup.rs b/src/discord/backup.rs new file mode 100644 index 0000000..9905baf --- /dev/null +++ b/src/discord/backup.rs @@ -0,0 +1,128 @@ +use crate::discord::api::DiscordAPI; +use serde::Serialize; +use serenity::all::{GetMessages, Message, MessageType, UserId}; +use tracing::{error, info, instrument}; + +#[instrument(skip(discord))] +pub async fn backup_user_votes(discord: &DiscordAPI, user_id: UserId) -> Option> { + let dm_channel = user_id.create_dm_channel(&discord.client.http).await; + + if let Err(err) = dm_channel { + error!( + "Failed to create DM channel! Error: {}", + err + ); + return None; + } + + let messages = dm_channel + .unwrap() + .messages(&discord.client.http, GetMessages::new()) + .await; + + if let Err(err) = messages { + error!( + "Failed to get messages from DM channel! Error: {}", + err + ); + return None; + } + + let messages: Vec = messages + .unwrap() + .iter() + .filter_map(|message| extract_vote(discord, user_id, message)) + .collect(); + + info!("Found {} votes", messages.len()); + + Some(messages) +} + +#[instrument(skip_all)] +fn extract_vote(discord: &DiscordAPI, user_id: UserId, message: &Message) -> Option { + if message.author.id != discord.own_user.id + || message.kind != MessageType::Regular + || message.embeds.is_empty() + { + return None; + } + + let embed = &message.embeds[0]; + let description = embed.description.clone(); + + if description.is_none() { + error!("No description on event!"); + return None; + } + + let description = description.unwrap(); + let embed_fields = &embed.fields; + let user_vote = match embed_fields + .iter() + .find(|field| field.name == "Voto") + .cloned() + { + Some(vote) => { + let comments = embed_fields + .iter() + .find(|field| field.name == "Comentários") + .map(|comment_field| comment_field.value.clone()); + + UserVote { + vote: vote.value, + comments, + } + } + None => { + // Fallback for embed-less reviews (backwards compatibility) + let vote = description + .lines() + .find(|line| line.starts_with("**Voto:** ")) + .map(|line| line.replace("**Voto:** ", "").trim().to_string()); + let comments = description + .lines() + .find(|line| line.starts_with("**Comentários:** ")) + .map(|line| line.replace("**Comentários:** ", "").trim().to_string()); + + if vote.is_none() { + error!("No vote found in description on an embed-less review!"); + return None; + } + + UserVote { + vote: vote.unwrap(), + comments, + } + } + }; + + Some(VoteRecord { + user_id, + title: embed.title.clone().unwrap_or_else(|| { + error!("No title on event"); + "No Title".to_string() + }), + url: embed.url.clone().unwrap_or_else(|| { + error!("No URL on event"); + "No URL".to_string() + }), + description, + user_vote, + }) +} + +#[derive(Serialize, Debug)] +pub struct VoteRecord { + pub user_id: UserId, + pub title: String, + pub url: String, + pub description: String, + pub user_vote: UserVote, +} + +#[derive(Serialize, Debug)] +pub struct UserVote { + pub vote: String, + pub comments: Option, +} diff --git a/src/discord/mod.rs b/src/discord/mod.rs index e5fdf85..1aa964d 100644 --- a/src/discord/mod.rs +++ b/src/discord/mod.rs @@ -1 +1,2 @@ pub mod api; +pub mod backup; diff --git a/src/main.rs b/src/main.rs index bd4d1cc..c867bdd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,11 +4,20 @@ use alertaemcena::api::*; use alertaemcena::config::env_loader::load_config; use alertaemcena::config::model::{Config, DebugConfig, EmojiConfig}; use alertaemcena::discord::api::{DiscordAPI, EventsThread}; +use alertaemcena::discord::backup::{backup_user_votes, VoteRecord}; use alertaemcena::tracing::setup_loki; +use chrono::Utc; +use futures::{future, TryFutureExt}; +use itertools::Itertools; use lazy_static::lazy_static; -use serenity::all::{ChannelId, GuildChannel, MessageType}; +use serenity::all::{ + ChannelId, GuildChannel, MessageType, UserId, +}; use std::collections::BTreeMap; use std::process::exit; +use tokio::fs; +use tokio::fs::File; +use tokio::io::AsyncWriteExt; use tracing::{debug, error, info, instrument, trace, warn}; lazy_static! { @@ -37,8 +46,15 @@ async fn main() { } } + let mut users_to_backup = Vec::new(); + if !config.debug_config.skip_artes { - run(&config, &discord, Category::Artes, config.artes_channel_id).await; + run(&config, &discord, Category::Artes, config.artes_channel_id) + .await + .iter() + .for_each(|u| { + users_to_backup.push(*u); + }) } run( @@ -47,7 +63,15 @@ async fn main() { Category::Teatro, config.teatro_channel_id, ) - .await; + .await + .iter() + .for_each(|u| { + if !users_to_backup.contains(u) { + users_to_backup.push(*u); + } + }); + + backup_votes(&discord, users_to_backup).await; } if let Some((controller, join_handle)) = loki_controller { @@ -56,38 +80,43 @@ async fn main() { } } -// No way will I match all variants of APIError -#[allow(clippy::unnecessary_unwrap)] -#[instrument(skip_all, fields(category = %category))] -async fn run(config: &Config, discord: &DiscordAPI, category: Category, channel_id: ChannelId) { +#[instrument(skip(config, discord, channel_id))] +async fn run( + config: &Config, + discord: &DiscordAPI, + category: Category, + channel_id: ChannelId, +) -> Vec { let guild = discord.get_guild(channel_id).await; let threads = discord.get_channel_threads(&guild, channel_id).await; + let mut users_with_reactions = Vec::new(); if !config.debug_config.skip_feature_reactions { - handle_reaction_features(discord, threads, &config.voting_emojis).await; + users_with_reactions = + handle_reaction_features(discord, threads, &config.voting_emojis).await; } info!("Handled reaction features"); if !config.gather_new_events { info!("Set to not gather new events"); - return; + return users_with_reactions; } let events = AgendaCulturalAPI::get_events_by_month(&category, config.debug_config.event_limit).await; - if events.is_err() { - let err = events.unwrap_err(); + if let Err(err) = events { error!("Failed getting events. Reason: {:?}", err); - return; + + return users_with_reactions; } let events = events.unwrap(); if events.is_empty() { error!("No events found"); - return; + return users_with_reactions; } let new_events = filter_new_events_by_thread(discord, &guild, events, channel_id).await; @@ -103,14 +132,86 @@ async fn run(config: &Config, discord: &DiscordAPI, category: Category, channel_ .await; info!("Finished sending new events for {}", category); + + users_with_reactions +} + +#[instrument(skip(discord))] +pub async fn backup_votes(discord: &DiscordAPI, vec: Vec) { + let vote_backups_folder = "vote_backups/"; + let vote_backup_file_path = format!( + "{}{}.json", + vote_backups_folder, + Utc::now().format("%Y_%m_%d") + ); + + fs::try_exists(vote_backups_folder) + .and_then(|exists| async move { + if exists { + Ok(()) + } else { + fs::create_dir(vote_backups_folder).await + } + }) + .unwrap_or_else(|e| { + error!("Failed to create vote backups folder! Error: {}", e); + }) + .await; + + match fs::try_exists(vote_backup_file_path.clone()).await { + Ok(exists) => { + if exists { + info!("Vote backup file already exists for today, skipping backup"); + return; + } + } + Err(e) => { + error!("Failed to check if vote backup file exists! Error: {}", e); + return; + } + } + + let user_votes: Vec = future::join_all( + vec.iter() + .map(|user_id| backup_user_votes(discord, *user_id)), + ) + .await + .into_iter() + .flatten() + .concat(); + + let backup_votes_file = File::create(&vote_backup_file_path).await; + + if let Err(err) = backup_votes_file { + error!( + "Failed to create vote backup file at {}! Error: {}", + vote_backup_file_path, + err + ); + return; + } + + let write_return = backup_votes_file.unwrap() + .write_all(&serde_json::to_vec_pretty(&user_votes).expect("Failed to serialize user votes")) + .await; + + if let Err(e) = write_return { + error!("Failed to write vote backup file! Error: {}", e); + return; + } + + info!("Vote backup file written to {}", vote_backup_file_path); } +/// Returns users who have used reaction features in the given threads #[instrument(skip(discord, threads, vote_emojis))] async fn handle_reaction_features( discord: &DiscordAPI, threads: Vec, vote_emojis: &[EmojiConfig; 5], -) { +) -> Vec { + let mut users_with_reactions = Vec::new(); + for thread in threads { if thread.thread_metadata.expect("Should be a thread!").locked { trace!("Ignoring locked thread (probably out-of-date)"); @@ -156,12 +257,21 @@ async fn handle_reaction_features( discord .send_privately_users_review(&message, vote_emojis) - .await; + .await + .iter() + .for_each(|u| { + if !users_with_reactions.contains(u) { + users_with_reactions.push(*u); + } + }); } } + + users_with_reactions } -#[instrument(skip(discord, new_events, emojis, debug_config), fields(new_events_count = %new_events.len()))] +#[instrument(skip(discord, new_events, emojis, debug_config), fields(new_events_count = %new_events.len() +))] async fn send_new_events( discord: &DiscordAPI, new_events: BTreeMap>, diff --git a/tests/discord_api_tests.rs b/tests/discord_api_tests.rs index d0df7ba..f126c81 100644 --- a/tests/discord_api_tests.rs +++ b/tests/discord_api_tests.rs @@ -12,6 +12,11 @@ mod discord { static ref token: String = env::var("DISCORD_TOKEN").expect("DISCORD_TOKEN not set"); static ref tester_token: String = env::var("DISCORD_TESTER_TOKEN").expect("DISCORD_TESTER_TOKEN not set"); + // Real user ID to test reactions and private messages + static ref user_id: u64 = env::var("DISCORD_USER_ID") + .expect("DISCORD_USER_ID not set") + .parse() + .expect("DISCORD_USER_ID is in a wrong format"); static ref channel_id: ChannelId = env::var("DISCORD_CHANNEL_ID") .expect("DISCORD_CHANNEL_ID not set") .parse() @@ -269,6 +274,30 @@ mod discord { assert_eq!(threads.pop().unwrap().name, thread_name); } + mod backup { + use super::{build_api, user_id}; + use alertaemcena::discord::backup::backup_user_votes; + use serenity::all::UserId; + + #[test_log::test(tokio::test)] + async fn should_backup_user_votes_from_dm() { + let api = build_api().await; + let discord_user_id = UserId::from(*user_id); + let backup = backup_user_votes(&api, discord_user_id).await; + + assert!(backup.is_some()); + + let votes = backup.unwrap(); + + assert!( + votes + .iter() + .any(|vote_record| vote_record.title == "O Auto da Barca do Inferno"), + "Didn't find the sent message in the backup!" + ); + } + } + mod end_to_end { use crate::discord::channel_id; use crate::discord::helpers::{build_api, generate_random_event, send_event};