Revision of check-urls.js

carbontwelve revised this gist 2 weeks ago. Go to revision

2 files changed, 136 insertions

README.md(file created)

		@@ -0,0 +1,19 @@
1	+	# Check URLs function for Hacker News and Lobsters
2	+
3	+	For use with my daily log link list as a way of identifying submissions to Hacker News and Lobsters so that I may link to them from the links detail page.
4	+
5	+	## Example Usage
6	+	```js
7	+	import { readFile } from "node:fs/promises";
8	+
9	+	const content = await readFile("urls.txt", "utf8");
10	+
11	+	const urls = content
12	+	.split("\n")
13	+	.map(x => x.trim())
14	+	.filter(Boolean);
15	+
16	+	const results = await checkUrls(urls, 5);
17	+
18	+	console.log(JSON.stringify(results, null, 2));
19	+	```

check-urls.js(file created)

		@@ -0,0 +1,117 @@
1	+	// --------------------
2	+	// Individual providers
3	+	// --------------------
4	+
5	+	async function getHackerNewsSubmissions(url) {
6	+	const endpoint =
7	+	`https://hn.algolia.com/api/v1/search` +
8	+	`?query=${encodeURIComponent(url)}` +
9	+	`&restrictSearchableAttributes=url` +
10	+	`&tags=story`;
11	+
12	+	const response = await fetch(endpoint);
13	+
14	+	if (!response.ok) {
15	+	throw new Error(`HN search failed: ${response.status}`);
16	+	}
17	+
18	+	const data = await response.json();
19	+
20	+	return (data.hits \|\| []).map(hit => ({
21	+	id: hit.objectID,
22	+	title: hit.title,
23	+	url: hit.url,
24	+	author: hit.author,
25	+	points: hit.points,
26	+	comments: hit.num_comments,
27	+	createdAt: hit.created_at,
28	+	discussionUrl: `https://news.ycombinator.com/item?id=${hit.objectID}`
29	+	}));
30	+	}
31	+
32	+	async function getLobstersSubmissions(url) {
33	+	try {
34	+	const endpoint =
35	+	`https://lobste.rs/search.json?q=${encodeURIComponent(url)}`;
36	+
37	+	const response = await fetch(endpoint);
38	+
39	+	if (!response.ok) {
40	+	return [];
41	+	}
42	+
43	+	const results = await response.json();
44	+
45	+	return (results \|\| [])
46	+	.filter(item => item.url === url)
47	+	.map(item => ({
48	+	id: item.short_id,
49	+	title: item.title,
50	+	url: item.url,
51	+	author: item.submitter_user?.username,
52	+	points: item.score,
53	+	comments: item.comment_count,
54	+	createdAt: item.created_at,
55	+	discussionUrl: `https://lobste.rs/s/${item.short_id}`
56	+	}));
57	+	} catch {
58	+	return [];
59	+	}
60	+	}
61	+
62	+	// --------------------
63	+	// Simple concurrency limiter
64	+	// --------------------
65	+
66	+	async function mapLimit(items, limit, mapper) {
67	+	const results = new Array(items.length);
68	+	let nextIndex = 0;
69	+
70	+	async function worker() {
71	+	while (true) {
72	+	const index = nextIndex++;
73	+
74	+	if (index >= items.length) {
75	+	return;
76	+	}
77	+
78	+	results[index] = await mapper(items[index], index);
79	+	}
80	+	}
81	+
82	+	const workers = Array.from(
83	+	{ length: Math.min(limit, items.length) },
84	+	worker
85	+	);
86	+
87	+	await Promise.all(workers);
88	+
89	+	return results;
90	+	}
91	+
92	+	// --------------------
93	+	// Main batch function
94	+	// --------------------
95	+
96	+	async function checkUrls(urls, concurrency = 5) {
97	+	return mapLimit(urls, concurrency, async url => {
98	+	const [hackerNews, lobsters] = await Promise.allSettled([
99	+	getHackerNewsSubmissions(url),
100	+	getLobstersSubmissions(url)
101	+	]);
102	+
103	+	return {
104	+	url,
105	+
106	+	hackerNews:
107	+	hackerNews.status === "fulfilled"
108	+	? hackerNews.value
109	+	: [],
110	+
111	+	lobsters:
112	+	lobsters.status === "fulfilled"
113	+	? lobsters.value
114	+	: []
115	+	};
116	+	});
117	+	}

Newer Older