Check URLs function for Hacker News and Lobsters

For use with my daily log link list as a way of identifying submissions to Hacker News and Lobsters so that I may link to them from the links detail page.

Example Usage

import { readFile } from "node:fs/promises";

const content = await readFile("urls.txt", "utf8");

const urls = content
  .split("\n")
  .map(x => x.trim())
  .filter(Boolean);

const results = await checkUrls(urls, 5);

console.log(JSON.stringify(results, null, 2));

check-urls.js · 2.5 KiB · JavaScript Raw

// -------------------- // Individual providers // -------------------- async function getHackerNewsSubmissions(url) { const endpoint = `https://hn.algolia.com/api/v1/search` + `?query=${encodeURIComponent(url)}` + `&restrictSearchableAttributes=url` + `&tags=story`; const response = await fetch(endpoint); if (!response.ok) { throw new Error(`HN search failed: ${response.status}`); } const data = await response.json(); return (data.hits || []).map(hit => ({ id: hit.objectID, title: hit.title, url: hit.url, author: hit.author, points: hit.points, comments: hit.num_comments, createdAt: hit.created_at, discussionUrl: `https://news.ycombinator.com/item?id=${hit.objectID}` })); } async function getLobstersSubmissions(url) { try { const endpoint = `https://lobste.rs/search.json?q=${encodeURIComponent(url)}`; const response = await fetch(endpoint); if (!response.ok) { return []; } const results = await response.json(); return (results || []) .filter(item => item.url === url) .map(item => ({ id: item.short_id, title: item.title, url: item.url, author: item.submitter_user?.username, points: item.score, comments: item.comment_count, createdAt: item.created_at, discussionUrl: `https://lobste.rs/s/${item.short_id}` })); } catch { return []; } } // -------------------- // Simple concurrency limiter // -------------------- async function mapLimit(items, limit, mapper) { const results = new Array(items.length); let nextIndex = 0; async function worker() { while (true) { const index = nextIndex++; if (index >= items.length) { return; } results[index] = await mapper(items[index], index); } } const workers = Array.from( { length: Math.min(limit, items.length) }, worker ); await Promise.all(workers); return results; } // -------------------- // Main batch function // -------------------- async function checkUrls(urls, concurrency = 5) { return mapLimit(urls, concurrency, async url => { const [hackerNews, lobsters] = await Promise.allSettled([ getHackerNewsSubmissions(url), getLobstersSubmissions(url) ]); return { url, hackerNews: hackerNews.status === "fulfilled" ? hackerNews.value : [], lobsters: lobsters.status === "fulfilled" ? lobsters.value : [] }; }); }

1	// --------------------
2	// Individual providers
3	// --------------------
4
5	async function getHackerNewsSubmissions(url) {
6	const endpoint =
7	`https://hn.algolia.com/api/v1/search` +
8	`?query=${encodeURIComponent(url)}` +
9	`&restrictSearchableAttributes=url` +
10	`&tags=story`;
11
12	const response = await fetch(endpoint);
13
14	if (!response.ok) {
15	throw new Error(`HN search failed: ${response.status}`);
16	}
17
18	const data = await response.json();
19
20	return (data.hits \|\| []).map(hit => ({
21	id: hit.objectID,
22	title: hit.title,
23	url: hit.url,
24	author: hit.author,
25	points: hit.points,
26	comments: hit.num_comments,
27	createdAt: hit.created_at,
28	discussionUrl: `https://news.ycombinator.com/item?id=${hit.objectID}`
29	}));
30	}
31
32	async function getLobstersSubmissions(url) {
33	try {
34	const endpoint =
35	`https://lobste.rs/search.json?q=${encodeURIComponent(url)}`;
36
37	const response = await fetch(endpoint);
38
39	if (!response.ok) {
40	return [];
41	}
42
43	const results = await response.json();
44
45	return (results \|\| [])
46	.filter(item => item.url === url)
47	.map(item => ({
48	id: item.short_id,
49	title: item.title,
50	url: item.url,
51	author: item.submitter_user?.username,
52	points: item.score,
53	comments: item.comment_count,
54	createdAt: item.created_at,
55	discussionUrl: `https://lobste.rs/s/${item.short_id}`
56	}));
57	} catch {
58	return [];
59	}
60	}
61
62	// --------------------
63	// Simple concurrency limiter
64	// --------------------
65
66	async function mapLimit(items, limit, mapper) {
67	const results = new Array(items.length);
68	let nextIndex = 0;
69
70	async function worker() {
71	while (true) {
72	const index = nextIndex++;
73
74	if (index >= items.length) {
75	return;
76	}
77
78	results[index] = await mapper(items[index], index);
79	}
80	}
81
82	const workers = Array.from(
83	{ length: Math.min(limit, items.length) },
84	worker
85	);
86
87	await Promise.all(workers);
88
89	return results;
90	}
91
92	// --------------------
93	// Main batch function
94	// --------------------
95
96	async function checkUrls(urls, concurrency = 5) {
97	return mapLimit(urls, concurrency, async url => {
98	const [hackerNews, lobsters] = await Promise.allSettled([
99	getHackerNewsSubmissions(url),
100	getLobstersSubmissions(url)
101	]);
102
103	return {
104	url,
105
106	hackerNews:
107	hackerNews.status === "fulfilled"
108	? hackerNews.value
109	: [],
110
111	lobsters:
112	lobsters.status === "fulfilled"
113	? lobsters.value
114	: []
115	};
116	});
117	}