diff --git a/api/routes/votd.js b/api/routes/votd.js index 7c37d0d..549553a 100644 --- a/api/routes/votd.js +++ b/api/routes/votd.js @@ -50,7 +50,7 @@ export default function votdApiRoute(app, db) { } catch (error) { res.send({ success: false, - message: `Error creating devotion entry: ${error}`, + message: `Error creating VOTD entry: ${error}`, }); } }); diff --git a/package-lock.json b/package-lock.json index 180c491..86915a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "xml2js": "^0.5.0" }, "engines": { - "node": "20.0.0", + "node": ">=20.18.1", "npm": ">=8.5.0" } }, diff --git a/routes/index.js b/routes/index.js index 1b8510b..4e60a56 100644 --- a/routes/index.js +++ b/routes/index.js @@ -4,83 +4,102 @@ import moment from "moment"; import xml2js from 'xml2js'; import { removeHtmlEntities } from '../app.js'; +const devotionCache = new Map(); + export default function applicationSiteRoutes(app) { app.get('/', async function (req, res) { return res.send(`DevoteMe-API\nConnection for all of the DevoteMe suite applications.\nDeveloped by Modular Software\nDocumentation: https://modularsoft.org/docs/products/devoteMe/`); }); app.get('/devotion/get', async function (req, res) { + const today = moment().format('YYYY-MM-DD'); + if (devotionCache.has(today)) { + console.log(`[devotion/get] Returning cached devotion for ${today}`); + return res.send(devotionCache.get(today)); + } + try { - console.log('[devotion/get] Fetching from intouchaustralia.org...'); - const response = await fetch('https://www.intouchaustralia.org/read/daily-devotions', { + const url = 'https://www.intouch.org/read/daily-devotions'; + console.log('[devotion/get] Fetching from intouch.org...'); + const response = await fetch(url, { headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept-Language': 'en-US,en;q=0.9', - } + 'Cache-Control': 'no-cache', + 'Pragma': 'no-cache' + }, + timeout: 15000 }); console.log(`[devotion/get] Response status: ${response.status} ${response.statusText}`); - console.log(`[devotion/get] Response headers:`, Object.fromEntries(response.headers.entries())); if (!response.ok) { const body = await response.text(); - console.log(`[devotion/get] Error response body (first 500 chars): ${body.substring(0, 500)}`); + console.log(`[devotion/get] Error response body snippet: ${body.substring(0, 200)}`); return res.status(502).send({ error: `Failed to fetch devotion from source (HTTP ${response.status})` }); } const html = await response.text(); - console.log(`[devotion/get] HTML length: ${html.length} chars`); - console.log(`[devotion/get] HTML snippet (first 500 chars): ${html.substring(0, 500)}`); + if (html.includes('Incapsula_Resource') && html.length < 1000) { + console.log('[devotion/get] Blocked by Incapsula challenge page'); + return res.status(502).send({ + error: 'Access denied by source security filter (WAF)' + }); + } const $ = cheerio.load(html); - const devotionTitle = $('h1').first().text().trim(); + let devotionTitle = $('h1').first().text().trim(); + if (!devotionTitle || devotionTitle.toLowerCase().includes('daily devotions')) { + devotionTitle = $('.title-4xl').first().text().trim() || + $('meta[property="og:title"]').attr('content') || + 'Daily Devotion'; + } console.log(`[devotion/get] Devotion title: "${devotionTitle}"`); - const date = moment(new Date()).format('Do MMMM YYYY'); - - // Log available selectors for debugging - const articleCount = $('article').length; - const jsScripturizeCount = $('article.js-scripturize').length; - const wysiwygCount = $('article.js-scripturize .wysiwyg').length; - console.log(`[devotion/get] Selector matches — article: ${articleCount}, article.js-scripturize: ${jsScripturizeCount}, .wysiwyg: ${wysiwygCount}`); + let devotionContent = $('article.js-scripturize .wysiwyg').find('p'); + if (devotionContent.length === 0) devotionContent = $('.wysiwyg p'); + if (devotionContent.length === 0) devotionContent = $('article p'); + if (devotionContent.length === 0) devotionContent = $('.content p'); - const devotionContent = $('article.js-scripturize .wysiwyg').find('p'); console.log(`[devotion/get] Found ${devotionContent.length} paragraph elements`); - const contentArray = devotionContent.map((i, el) => $(el).text().trim()).get(); - console.log(`[devotion/get] Content array (${contentArray.length} items):`, JSON.stringify(contentArray.map(s => s.substring(0, 80)))); + const contentArray = devotionContent.map((i, el) => removeHtmlEntities($(el).text().trim())).get() + .filter(text => text.length > 0); if (contentArray.length === 0) { - // Log all article content for debugging - const allArticles = $('article').map((i, el) => $(el).attr('class')).get(); - console.log(`[devotion/get] All article classes:`, allArticles); - const allH1 = $('h1').map((i, el) => $(el).text().trim()).get(); - console.log(`[devotion/get] All h1 elements:`, allH1); - + console.log(`[devotion/get] Unable to parse content. HTML snippet: ${html.substring(0, 500)}`); return res.status(502).send({ - error: 'Unable to parse devotion content from source — the page structure may have changed' + error: 'Unable to parse devotion content from source' }); } - const devotionReading = contentArray.splice(0, 1)[0]; - const bibleInOneYear = contentArray.splice(-1, 1)[0]; - console.log(`[devotion/get] Reading: "${devotionReading}"`); - console.log(`[devotion/get] Bible in One Year: "${bibleInOneYear}"`); + const devotionReading = contentArray.shift(); + + let bibleInOneYear = null; + if (contentArray.length > 0) { + const lastItem = contentArray[contentArray.length - 1]; + if (lastItem.toLowerCase().includes('bible in one year')) { + bibleInOneYear = contentArray.pop(); + console.log(`[devotion/get] Extracted Bible in One Year: "${bibleInOneYear}"`); + } + } const devotion = { title: devotionTitle, - date: date, + date: moment().format('Do MMMM YYYY'), reading: devotionReading, - content: contentArray, + content: contentArray.join('\n\n'), + paragraphs: contentArray, bibleInOneYear: bibleInOneYear ? bibleInOneYear.replace(/^Bible in One Year:\s+/i, '') : null, - credit: "From In Touch Australia (https://www.intouchaustralia.org/read/daily-devotions)" + credit: "From In Touch Ministries (https://www.intouch.org/read/daily-devotions)" }; console.log(`[devotion/get] Success — returning devotion: "${devotionTitle}"`); + devotionCache.set(today, devotion); return res.send(devotion); } catch (error) {