Show if blocked in /api/stats

This is experimental and is not listed in /api/stats:features yet.

src/lib/collectors.js

@@ -2,23 +2,29 @@ const constants = require("./constants")
 const {request} = require("./utils/request")
 const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache} = require("./cache")
-require("./testimports")(constants, request, extractSharedData, RequestCache)
+const RequestHistory = require("./structures/RequestHistory")
+require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
 /** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
 const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
+const history = new RequestHistory(["user", "timeline", "post"])
 
 function fetchUser(username) {
 	return requestCache.getOrFetch("user/"+username, () => {
 		return request(`https://www.instagram.com/${username}/`).then(res => {
-			if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
-			else if (res.status === 404) throw constants.symbols.NOT_FOUND
-			else return res.text().then(text => {
+			if (res.status === 302) {
+				history.report("user", false)
+				throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
+			} else if (res.status === 404) {
+				throw constants.symbols.NOT_FOUND
+			} else return res.text().then(text => {
 				// require down here or have to deal with require loop. require cache will take care of it anyway.
 				// User -> Timeline -> TimelineImage -> collectors -/> User
 				const User = require("./structures/User")
 				const sharedData = extractSharedData(text)
 				const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
+				history.report("user", true)
 				return user
 			})
 		})
@@ -40,10 +46,16 @@ function fetchTimelinePage(userID, after) {
 	}))
 	return requestCache.getOrFetchPromise("page/"+after, () => {
 		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
-			if (!root.data) console.error("missing data:", root) //todo: please make this better.
-			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
-			const timeline = root.data.user.edge_owner_to_timeline_media
-			return timeline
+			if (!root.data) {
+				history.report("timeline", false)
+				console.error("missing data from timeline request, 429?", root) //todo: please make this better.
+				throw new Error("missing data from timeline request, 429?")
+			} else {
+				/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
+				const timeline = root.data.user.edge_owner_to_timeline_media
+				history.report("timeline", true)
+				return timeline
+			}
 		})
 	})
 }
@@ -89,13 +101,20 @@ function fetchShortcodeData(shortcode) {
 	p.set("variables", JSON.stringify({shortcode}))
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
 		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
+			if (!root.data) {
+				history.report("post", false)
+				console.error("missing data from post request, 429?", root) //todo: please make this better.
+				throw new Error("missing data from post request, 429?")
 			/** @type {import("./types").TimelineEntryN3} */
-			const data = root.data.shortcode_media
-			if (data == null) {
-				// the thing doesn't exist
-				throw constants.symbols.NOT_FOUND
 			} else {
-				return data
+				const data = root.data.shortcode_media
+				if (data == null) {
+					// the thing doesn't exist
+					throw constants.symbols.NOT_FOUND
+				} else {
+					history.report("post", true)
+					return data
+				}
 			}
 		})
 	})
@@ -108,3 +127,4 @@ module.exports.fetchShortcodeData = fetchShortcodeData
 module.exports.requestCache = requestCache
 module.exports.timelineEntryCache = timelineEntryCache
 module.exports.getOrFetchShortcode = getOrFetchShortcode
+module.exports.history = history

src/lib/constants.js

@@ -17,6 +17,7 @@ let constants = {
 
 	// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
 	external: {
+		user_query_hash: "c9100bf9110dd6361671f113dd02e7d6",
 		timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
 		shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90",
 		timeline_fetch_first: 12,

src/lib/structures/RequestHistory.js

@@ -0,0 +1,37 @@
+class RequestHistory {
+	/**
+	 * @param {string[]} tracked list of things that can be tracked
+	 */
+	constructor(tracked) {
+		this.tracked = new Set(tracked)
+		/** @type {Map<string, {lastRequestAt: number | null, lastRequestSuccessful: boolean | null}>} */
+		this.store = new Map()
+		for (const key of tracked) {
+			this.store.set(key, {
+				lastRequestAt: null,
+				lastRequestSuccessful: null
+			})
+		}
+	}
+
+	/**
+	 * @param {string} key
+	 * @param {boolean} success
+	 */
+	report(key, success) {
+		if (!this.tracked.has(key)) throw new Error(`Trying to report key ${key}, but is not tracked`)
+		const entry = this.store.get(key)
+		entry.lastRequestAt = Date.now()
+		entry.lastRequestSuccessful = success
+	}
+
+	export() {
+		const result = {}
+		for (const key of this.store.keys()) {
+			result[key] = this.store.get(key)
+		}
+		return result
+	}
+}
+
+module.exports = RequestHistory