import Parser from 'rss-parser' import { Feed } from 'feed' import Handlebars from 'handlebars/runtime' import template from './templates/default.precompiled' import * as striptags from 'striptags' /** * Extra Handlerbars template helpers */ Handlebars.registerHelper('isRowElemN', function(index, rowItems, n, options) { return index % rowItems == n ? options.fn(this) : options.inverse(this) }) /** * Handle CRON jobs * Where information is gathered and HTML and RSS is generated. */ addEventListener('scheduled', event => { event.waitUntil(handleScheduled()) }) /** * Serve the existing generated elements. * CACHE is used to speed up the operation. */ addEventListener('fetch', event => { event.respondWith(handleRequest(event.request)) }) /** * Deliver aggregated content according to the formats requested * @param {Request} request * @returns Response */ async function handleRequest(request) { const cacheUrl = new URL(request.url) const cacheKey = new Request(cacheUrl.toString(), request) const cache = caches.default const cacheMaxAge = CACHE_MAX_AGE || 3600 let response = await cache.match(cacheKey) if (response) return response const path = new URL(request.url).pathname if (path === '/') { let content = await WORKER_PLANET_STORE.get('html') response = new Response(content, { headers: { 'content-type': 'text/html;charset=UTF-8', 'Cache-Control': `max-age=${cacheMaxAge}`, }, }) } else if (path === '/rss') { let content = await WORKER_PLANET_STORE.get('rss') response = new Response(content, { headers: { 'content-type': 'application/rss+xml', 'Cache-Control': `max-age=${cacheMaxAge}`, }, }) } else if (path === '/atom') { let content = await WORKER_PLANET_STORE.get('atom') response = new Response(content, { headers: { 'content-type': 'application/atom+xml', 'Cache-Control': `max-age=${cacheMaxAge}`, }, }) } else { return new Response('', { status: 404 }) } await cache.put(cacheKey, response.clone()) return response } /** * Fetch all source feeds and generate the aggregated content */ async function handleScheduled() { let feeds = FEEDS.split(',') let content = [] let sources = [] let promises = [] for (let url of feeds) { promises.push(fetchAndHydrate(url)) } const results = await Promise.allSettled(promises) for (let [index, result] of results.entries()) { if (result.status == 'fulfilled') { let posts = result.value let title = posts[0].source_title let link = posts[0].source_link let name = title != '' ? title : new URL(link).host sources.push({ name, link }) content.push(...posts) } else { console.log(`Failed to fetch ${feeds[index]}`) console.log(result.reason) } } //sort all the elements chronologically (recent first) content.sort((a, b) => { let aDate = new Date(a.isoDate) let bDate = new Date(b.isoDate) if (aDate < bDate) { return 1 } else if (aDate === bDate) { return 0 } else { return -1 } }) if (content.length > MAX_SIZE) { content = content.slice(0, MAX_SIZE) } // Generate feed let feed = createFeed(content) let html = createHTML(content, sources) // Store await WORKER_PLANET_STORE.put('rss', feed.rss2()) await WORKER_PLANET_STORE.put('atom', feed.atom1()) await WORKER_PLANET_STORE.put('html', html) } /** * Take a feed URL, fetch all items and attach source information * @param {String} feed The URL of the feed to be fetched and parsed * @returns Array containing all the feed items parsed by rss-parser */ async function fetchAndHydrate(feed) { console.log(`[fetchAndHydrate] start to fetch feed: ${feed}`) let resp = await fetch(feed) console.log(`[fetchAndHydrate] response: ${resp.status}`) let parser = new Parser() let content = await resp.text() let contentFeed = await parser.parseString(content) for (let item of contentFeed.items) { item.source_title = contentFeed.title item.source_link = contentFeed.link if ('content:encoded' in item) { item.content = item['content:encoded'] } } console.log( `[fetchAndHydrate] Finished fetch feed: ${feed}. ${contentFeed.items.length} items gathered`, ) return contentFeed.items } /** * Builds a feed object from the provided items * @param {Array} items parsed by rss-parser * @return Feed object created by feed */ function createFeed(items) { console.log(`[createFeed] start building the aggregated feed`) const feed = new Feed({ title: TITLE, description: DESCRIPTION, id: CUSTOM_URL, link: CUSTOM_URL, }) for (let item of items) { feed.addItem({ title: item.title, id: item.guid, link: item.link, description: item.contentSnippet, content: item.content, author: [ { name: item.creator, email: '', link: item.source_link, }, ], contributor: [], date: new Date(item.isoDate), }) } console.log(`[createFeed] Finished building the aggregated feed`) return feed } /** * Generate the HTML page with the aggregated contents * @param {Array} items parsed by rss-parser * @returns String with HTML page containing the parsed contents */ function createHTML(items, sources) { console.log(`[createHTML] building the HTML document`) let template = Handlebars.templates['default'] let dateFormatter = new Intl.DateTimeFormat('pt-PT', { timeZone: 'UTC' }) for (let item of items) { let shortdescription = striptags(item.content).substring(0, 250) item.description = shortdescription ? shortdescription + ' [...]' : '' item.formattedDate = item.pubDate ? dateFormatter.format(new Date(item.pubDate)) : '' } return template({ items: items, sources: sources, page_title: TITLE, page_description: DESCRIPTION, }) }