Skip to content

Fix devotion processing and logging #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
benrobson wants to merge 4 commits into
base: claude/debug-devotion-endpoint-pDO9n
Choose a base branch
from
Open

Fix devotion processing and logging #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1ef84d1
Fix devotion command processing and improve logging
google-labs-jules[bot] Feb 16, 2026
66af4ce
Fix devotion command processing by switching to global source and imp…
google-labs-jules[bot] Feb 16, 2026
9df61b6
Final fix for devotion processing: multi-source, caching, and bot-fil…
google-labs-jules[bot] Feb 16, 2026
e792a3a
Fix devotion processing: bypass WAF with Facebook UA and implement ca…
google-labs-jules[bot] Feb 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/routes/votd.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ export default function votdApiRoute(app, db) {
} catch (error) {
res.send({
success: false,
message: `Error creating devotion entry: ${error}`,
message: `Error creating VOTD entry: ${error}`,
});
}
});
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

87 changes: 53 additions & 34 deletions routes/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,83 +4,102 @@ import moment from "moment";
import xml2js from 'xml2js';
import { removeHtmlEntities } from '../app.js';

const devotionCache = new Map();

export default function applicationSiteRoutes(app) {
app.get('/', async function (req, res) {
return res.send(`DevoteMe-API\nConnection for all of the DevoteMe suite applications.\nDeveloped by Modular Software\nDocumentation: https://modularsoft.org/docs/products/devoteMe/`);
});

app.get('/devotion/get', async function (req, res) {
const today = moment().format('YYYY-MM-DD');
if (devotionCache.has(today)) {
console.log(`[devotion/get] Returning cached devotion for ${today}`);
return res.send(devotionCache.get(today));
}

try {
console.log('[devotion/get] Fetching from intouchaustralia.org...');
const response = await fetch('https://www.intouchaustralia.org/read/daily-devotions', {
const url = 'https://www.intouch.org/read/daily-devotions';
console.log('[devotion/get] Fetching from intouch.org...');
const response = await fetch(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent': 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'en-US,en;q=0.9',
}
'Cache-Control': 'no-cache',
'Pragma': 'no-cache'
},
timeout: 15000
});

console.log(`[devotion/get] Response status: ${response.status} ${response.statusText}`);
console.log(`[devotion/get] Response headers:`, Object.fromEntries(response.headers.entries()));

if (!response.ok) {
const body = await response.text();
console.log(`[devotion/get] Error response body (first 500 chars): ${body.substring(0, 500)}`);
console.log(`[devotion/get] Error response body snippet: ${body.substring(0, 200)}`);
return res.status(502).send({
error: `Failed to fetch devotion from source (HTTP ${response.status})`
});
}

const html = await response.text();
console.log(`[devotion/get] HTML length: ${html.length} chars`);
console.log(`[devotion/get] HTML snippet (first 500 chars): ${html.substring(0, 500)}`);
if (html.includes('Incapsula_Resource') && html.length < 1000) {
console.log('[devotion/get] Blocked by Incapsula challenge page');
return res.status(502).send({
error: 'Access denied by source security filter (WAF)'
});
}

const $ = cheerio.load(html);

const devotionTitle = $('h1').first().text().trim();
let devotionTitle = $('h1').first().text().trim();
if (!devotionTitle || devotionTitle.toLowerCase().includes('daily devotions')) {
devotionTitle = $('.title-4xl').first().text().trim() ||
$('meta[property="og:title"]').attr('content') ||
'Daily Devotion';
}
console.log(`[devotion/get] Devotion title: "${devotionTitle}"`);

const date = moment(new Date()).format('Do MMMM YYYY');

// Log available selectors for debugging
const articleCount = $('article').length;
const jsScripturizeCount = $('article.js-scripturize').length;
const wysiwygCount = $('article.js-scripturize .wysiwyg').length;
console.log(`[devotion/get] Selector matches — article: ${articleCount}, article.js-scripturize: ${jsScripturizeCount}, .wysiwyg: ${wysiwygCount}`);
let devotionContent = $('article.js-scripturize .wysiwyg').find('p');
if (devotionContent.length === 0) devotionContent = $('.wysiwyg p');
if (devotionContent.length === 0) devotionContent = $('article p');
if (devotionContent.length === 0) devotionContent = $('.content p');

const devotionContent = $('article.js-scripturize .wysiwyg').find('p');
console.log(`[devotion/get] Found ${devotionContent.length} paragraph elements`);

const contentArray = devotionContent.map((i, el) => $(el).text().trim()).get();
console.log(`[devotion/get] Content array (${contentArray.length} items):`, JSON.stringify(contentArray.map(s => s.substring(0, 80))));
const contentArray = devotionContent.map((i, el) => removeHtmlEntities($(el).text().trim())).get()
.filter(text => text.length > 0);

if (contentArray.length === 0) {
// Log all article content for debugging
const allArticles = $('article').map((i, el) => $(el).attr('class')).get();
console.log(`[devotion/get] All article classes:`, allArticles);
const allH1 = $('h1').map((i, el) => $(el).text().trim()).get();
console.log(`[devotion/get] All h1 elements:`, allH1);

console.log(`[devotion/get] Unable to parse content. HTML snippet: ${html.substring(0, 500)}`);
return res.status(502).send({
error: 'Unable to parse devotion content from source — the page structure may have changed'
error: 'Unable to parse devotion content from source'
});
}

const devotionReading = contentArray.splice(0, 1)[0];
const bibleInOneYear = contentArray.splice(-1, 1)[0];
console.log(`[devotion/get] Reading: "${devotionReading}"`);
console.log(`[devotion/get] Bible in One Year: "${bibleInOneYear}"`);
const devotionReading = contentArray.shift();

let bibleInOneYear = null;
if (contentArray.length > 0) {
const lastItem = contentArray[contentArray.length - 1];
if (lastItem.toLowerCase().includes('bible in one year')) {
bibleInOneYear = contentArray.pop();
console.log(`[devotion/get] Extracted Bible in One Year: "${bibleInOneYear}"`);
}
}

const devotion = {
title: devotionTitle,
date: date,
date: moment().format('Do MMMM YYYY'),
reading: devotionReading,
content: contentArray,
content: contentArray.join('\n\n'),
paragraphs: contentArray,
bibleInOneYear: bibleInOneYear ? bibleInOneYear.replace(/^Bible in One Year:\s+/i, '') : null,
credit: "From In Touch Australia (https://www.intouchaustralia.org/read/daily-devotions)"
credit: "From In Touch Ministries (https://www.intouch.org/read/daily-devotions)"
};

console.log(`[devotion/get] Success — returning devotion: "${devotionTitle}"`);
devotionCache.set(today, devotion);
return res.send(devotion);

} catch (error) {
Expand Down