From 305d7f6a9fed8e6a2f2b83759f9d77f0dad45f64 Mon Sep 17 00:00:00 2001 From: coomdev Date: Sat, 29 Jan 2022 21:01:45 +0100 Subject: [PATCH] WIP native browser extensions --- build.js | 5 +- main.d.ts | 6 +- package-lock.json | 15 +++- package.json | 3 +- src/Components/Embedding.svelte | 27 ++++--- src/background.ts | 34 +++++++++ src/filehosts.ts | 6 +- src/main.ts | 45 ++--------- src/platform.ts | 130 ++++++++++++++++++++++++++++++++ src/pngv3.ts | 1 - src/pomf.ts | 8 +- src/thirdeye.ts | 12 +-- src/utils.ts | 25 ++++-- 13 files changed, 245 insertions(+), 72 deletions(-) create mode 100644 src/background.ts create mode 100644 src/platform.ts diff --git a/build.js b/build.js index d3d1c5b..929b661 100644 --- a/build.js +++ b/build.js @@ -20,7 +20,10 @@ let rev = +res.stdout; bundle: true, outfile: "./dist/main.js", define: { - global: 'window' + global: 'window', + execution_mode: JSON.stringify(process.argv[2] || 'userscript'), + isBackground: JSON.stringify('false'), + BUILD_VERSION: JSON.stringify([0, rev]) }, inject: ['./esbuild.inject.js'], plugins: [ diff --git a/main.d.ts b/main.d.ts index fc3bd97..a8e3f98 100644 --- a/main.d.ts +++ b/main.d.ts @@ -16,4 +16,8 @@ declare module 'blockhash' { }, bits: number, method: number) => string; } -declare const QR: any; \ No newline at end of file +declare const QR: any; +declare const BUILD_VERSION: [number, number]; +declare const execution_mode: 'userscript' | 'chrome_api' | 'ff_api'; +declare const isBackground: boolean; +declare const chrome: typeof browser; diff --git a/package-lock.json b/package-lock.json index 2dbac14..f985eda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,8 @@ "svelte": "^3.44.3", "svelte-check": "^2.2.11", "svelte-preprocess": "^4.10.1", - "typescript": "^4.5.4" + "typescript": "^4.5.4", + "web-ext-types": "^3.2.1" } }, "node_modules/@babel/code-frame": { @@ -4658,6 +4659,12 @@ "extsprintf": "^1.2.0" } }, + "node_modules/web-ext-types": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", + "integrity": "sha512-oQZYDU3W8X867h8Jmt3129kRVKklz70db40Y6OzoTTuzOJpF/dB2KULJUf0txVPyUUXuyzV8GmT3nVvRHoG+Ew==", + "dev": true + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -8003,6 +8010,12 @@ "extsprintf": "^1.2.0" } }, + "web-ext-types": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz", + "integrity": "sha512-oQZYDU3W8X867h8Jmt3129kRVKklz70db40Y6OzoTTuzOJpF/dB2KULJUf0txVPyUUXuyzV8GmT3nVvRHoG+Ew==", + "dev": true + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 96f60bd..28baae2 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "svelte": "^3.44.3", "svelte-check": "^2.2.11", "svelte-preprocess": "^4.10.1", - "typescript": "^4.5.4" + "typescript": "^4.5.4", + "web-ext-types": "^3.2.1" }, "browser": { "node:buffer": "buffer", diff --git a/src/Components/Embedding.svelte b/src/Components/Embedding.svelte index 43dc2fb..7d84783 100644 --- a/src/Components/Embedding.svelte +++ b/src/Components/Embedding.svelte @@ -4,10 +4,8 @@ import { beforeUpdate, tick } from 'svelte' import type { EmbeddedFile } from '../main' import { createEventDispatcher } from 'svelte' - import { GM_head, headerStringToObject } from '../requests' - import { text } from 'svelte/internal' - import App from './App.svelte' import { Buffer } from 'buffer' + import { getHeaders, Platform } from '../platform' export const dispatch = createEventDispatcher() @@ -49,8 +47,9 @@ const thumb = file.thumbnail || file.data let type: FileTypeResult | undefined - if (typeof thumb != 'string') { - type = await fileTypeFromBuffer(thumb) + if (typeof thumb != "string") { + let buff = Buffer.isBuffer(thumb) ? thumb : await thumb(); + type = await fileTypeFromBuffer(buff) if ( !type && file.filename.endsWith('.txt') && @@ -58,12 +57,15 @@ ) { type = { ext: 'txt', mime: 'text/plain' } as any } - content = new Blob([thumb], { type: type?.mime }) + content = new Blob([buff], { type: type?.mime }) url = URL.createObjectURL(content) if (!type) return } else { - let head = headerStringToObject(await GM_head(thumb, undefined)) - type = { ext: '' as any, mime: head['content-type'].split(';')[0].trim() as any } + let head = await getHeaders(thumb) + type = { + ext: '' as any, + mime: head['content-type'].split(';')[0].trim() as any, + } } ftype = type.mime isVideo = type.mime.startsWith('video/') @@ -125,8 +127,11 @@ } else { url = file.data furl = file.data - let head = headerStringToObject(await GM_head(file.data, undefined)) - type = { ext: '' as any, mime: head['content-type'].split(';')[0].trim() as any } + let head = await getHeaders(file.data) + type = { + ext: '' as any, + mime: head['content-type'].split(';')[0].trim() as any, + } } if (!type) return isVideo = type.mime.startsWith('video/') @@ -191,7 +196,7 @@ ev.preventDefault() if (isNotChrome) { window.open(src, '_blank') - } else await GM.openInTab(src, { active: false, insert: true }) + } else await Platform.openInTab(src, { active: false, insert: true }) } } diff --git a/src/background.ts b/src/background.ts new file mode 100644 index 0000000..0716301 --- /dev/null +++ b/src/background.ts @@ -0,0 +1,34 @@ +import { Platform } from "./platform"; + +const obj = execution_mode == "chrome_api" ? chrome : browser; +type Methods = { + // eslint-disable-next-line @typescript-eslint/ban-types + [k in Exclude]: T[k] extends Function ? T[k] : never; +}; + +obj.webRequest.onBeforeRequest.addListener((details) => { + const redirectUrl = details.url; + if (!redirectUrl.startsWith("https://loli.piss/")) + return; + const m = redirectUrl.match(/https:\/\/loli.piss\/(?.*?)(?\/.*)\/(?.*)\/(?.*)/); + if (!m) + return; + const { domain, path, start, end } = m.groups!; + return { + redirectUrl: `https://${domain}${path}`, + requestHeaders: [{ + name: 'range', + value: `bytes=${start}-${end}` + }] + } as browser.webRequest.BlockingResponse; +}, { urls: ['*://loli.piss/*'] }, ['blocking']); + +obj.runtime.onConnect.addListener((c) => { + c.onMessage.addListener(async obj => { + const { id, name, args } = obj as {id: number, name: keyof Methods, args: Parameters]>}; + const res = await Platform[name](...args); + c.postMessage({ + id, res + }); + }); +}); \ No newline at end of file diff --git a/src/filehosts.ts b/src/filehosts.ts index 67de395..276d706 100644 --- a/src/filehosts.ts +++ b/src/filehosts.ts @@ -1,4 +1,4 @@ -import { GM_fetch } from "./requests"; +import { ifetch } from "./platform"; function parseForm(data: object) { const form = new FormData(); @@ -14,7 +14,7 @@ export const lolisafe = (domain: string, serving = domain) => ({ domain, serving, async uploadFile(f: Blob) { - const resp = await GM_fetch(`https://${domain}/api/upload`, { + const resp = await ifetch(`https://${domain}/api/upload`, { headers: { accept: "application/json", }, @@ -33,7 +33,7 @@ export const catbox = (domain: string, serving: string) => ({ domain, serving, async uploadFile(inj: Blob) { - const resp = await GM_fetch(`https://${domain}/user/api.php`, { + const resp = await ifetch(`https://${domain}/user/api.php`, { method: 'POST', body: parseForm({ reqtype: 'fileupload', diff --git a/src/main.ts b/src/main.ts index bc73666..decb47b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,8 +9,6 @@ import jpg, { convertToPng } from "./jpg"; import thirdeye from "./thirdeye"; import pomf from "./pomf"; -import { GM_fetch, GM_head, headerStringToObject } from "./requests"; - import App from "./Components/App.svelte"; import ScrollHighlighter from "./Components/ScrollHighlighter.svelte"; import PostOptions from "./Components/PostOptions.svelte"; @@ -23,6 +21,7 @@ import { buildPeeFile, fireNotification } from "./utils"; import { fileTypeFromBuffer } from "file-type"; import { getQueryProcessor, QueryProcessor } from "./websites"; import { lolisafe } from "./filehosts"; +import { ifetch, Platform, streamRemote, supportedAltDomain } from "./platform"; export interface ImageProcessor { skip?: true; @@ -49,38 +48,10 @@ appState.subscribe(v => { cappState = v; }); -// most pngs are encoded with 65k idat chunks -async function* streamRemote(url: string, chunkSize = 72 * 1024, fetchRestOnNonCanceled = true) { - const headers = await GM_head(url); - const h = headerStringToObject(headers); - const size = +h['content-length']; - let ptr = 0; - let fetchSize = chunkSize; - while (ptr != size) { - //console.log('doing a fetch of ', url, ptr, ptr + fetchSize - 1); - const res = await GM_fetch(url, { headers: { range: `bytes=${ptr}-${ptr + fetchSize - 1}` } }) as any as Tampermonkey.Response; - const obj = headerStringToObject(res.responseHeaders); - if (!('content-length' in obj)) { - console.warn("no content lenght???", url); - break; - } const len = +obj['content-length']; - ptr += len; - if (fetchRestOnNonCanceled) - fetchSize = size; - const val = Buffer.from(await (res as any).arrayBuffer()); - const e = (yield val) as boolean; - //console.log('yeieledd, a', e); - if (e) { - break; - } - } - //console.log("streaming ended, ", ptr, size); -} - type EmbeddedFileWithPreview = { page?: { title: string, url: string }; // can be a booru page source?: string; // can be like a twitter post this was posted in originally - thumbnail: Buffer; + thumbnail: string | Buffer; filename: string; data: EmbeddedFileWithoutPreview['data'] | ((lisn?: EventTarget) => Promise); }; @@ -88,7 +59,7 @@ type EmbeddedFileWithPreview = { type EmbeddedFileWithoutPreview = { page: undefined; source: undefined; - thumbnail: undefined; + thumbnail?: string; filename: string; data: string | Buffer; }; @@ -156,12 +127,12 @@ const processPost = async (post: HTMLDivElement) => { const versionCheck = async () => { const [lmajor, lminor] = - (await (await GM_fetch("https://git.coom.tech/coomdev/PEE/raw/branch/%e4%b8%ad%e5%87%ba%e3%81%97/main.meta.js")) + (await (await ifetch("https://git.coom.tech/coomdev/PEE/raw/branch/%e4%b8%ad%e5%87%ba%e3%81%97/main.meta.js")) .text()) .split('\n') .filter(e => e.includes("// @version"))[0].match(/.*version\s+(.*)/)![1].split('.') .map(e => +e); - const [major, minor] = GM.info.script.version.split('.').map(e => +e); + const [major, minor] = BUILD_VERSION; if (major < lmajor || (major == lmajor && minor < lminor)) { fireNotification("info", `Last PEE version is ${lmajor}.${lminor}, you're on ${major}.${minor}`); } @@ -183,7 +154,7 @@ const scrapeBoard = async (self: HTMLButtonElement) => { self.disabled = true; self.textContent = "Searching..."; const boardname = location.pathname.match(/\/(.*)\//)![1]; - const res = await GM_fetch(`https://a.4cdn.org/${boardname}/threads.json`); + const res = await ifetch(`https://a.4cdn.org/${boardname}/threads.json`); const pages = await res.json() as Page[]; type Page = { threads: Thread[] } type Thread = { no: number; posts: Post[] }; @@ -197,7 +168,7 @@ const scrapeBoard = async (self: HTMLButtonElement) => { .map(e => e.no) .map(async id => { try { - const res = await GM_fetch(`https://a.4cdn.org/${boardname}/thread/${id}.json`); + const res = await ifetch(`https://a.4cdn.org/${boardname}/thread/${id}.json`); return await res.json() as Thread; } catch { return undefined; @@ -391,7 +362,7 @@ const startup = async (is4chanX = true) => { document.addEventListener('4chanXInitFinished', () => startup(true)); document.addEventListener('4chanParsingDone', () => startup(false), { once: true }); -if (GM.info.script.matches.slice(2).some(m => m.includes(location.host))) { +if (supportedAltDomain(location.host)) { window.addEventListener('load', () => { startup(false); }, { once: true }); diff --git a/src/platform.ts b/src/platform.ts new file mode 100644 index 0000000..117e73a --- /dev/null +++ b/src/platform.ts @@ -0,0 +1,130 @@ +import { Buffer } from 'ts-ebml/lib/tools'; +import 'web-ext-types'; +import { GM_fetch, GM_head, headerStringToObject } from './requests'; + +let port: browser.runtime.Port; +const lqueue: ((e: any) => boolean)[] = []; + +if (execution_mode != 'userscript' && !isBackground) { + port = browser.runtime.connect(); + port.onMessage.addListener((e: any) => { + const k = lqueue.map(f => f(e)); + for (let i = k.length - 1; i != -1; --i) { + if (k[i]) + lqueue.splice(i, 1); + } + }); +} + +let gid = 0; +const bridge = V>(name: string, f: T) => { + if (execution_mode != 'userscript' && !isBackground) + return f; + return (...args: U) => { + const id = gid++; + const prom = new Promise(_ => { + lqueue.push((e: any) => { + if (e.id != id) + return false; + _(e.res); + return true; + }); + port.postMessage({ + id, name, args + }); + }); + return prom; + }; +}; + +// eslint-disable-next-line @typescript-eslint/ban-types +const Bridged = (ctor: any) => { + const keys = Object.getOwnPropertyNames(ctor).filter(k => typeof ctor[k] == "function"); + for (const k of keys) + ctor[k] = bridge(k, ctor[k]); +}; + +export function supportedAltDomain(s: string) { + if (execution_mode == 'userscript') + return GM.info.script.matches.slice(2).some(m => m.includes(s)); + return false; +} + +// Used to call background-only APIs from content scripts +@Bridged +export class Platform { + static async openInTab(src: string, opts: { active: boolean, insert: boolean }) { + if (execution_mode == 'userscript') { + return GM.openInTab(src, opts); + } + const obj = execution_mode == "chrome_api" ? chrome : browser; + if (execution_mode == 'chrome_api') { + let i: number | undefined; + if (opts.insert) + i = (await obj.tabs.getCurrent()).index + 1; + return obj.tabs.create({ active: opts.active, url: src, index: i }); + } + } +} + +export async function getHeaders(s: string) { + if (execution_mode == 'userscript') + return headerStringToObject(await GM_head(s)); + const res = await fetch(s, { + method: "HEAD" + }); + return [...res.headers.entries()].reduce((a, b) => (a[b[0]] = b[1], a), {} as ReturnType); +} + +export async function ifetch(...[url, opt, lisn]: [...Parameters, EventTarget?]): ReturnType { + if (execution_mode != "userscript") + return fetch(url, opt); + return GM_fetch(url, opt, lisn); +} + +// most pngs are encoded with 65k idat chunks +export async function* streamRemote(url: string, chunkSize = 72 * 1024, fetchRestOnNonCanceled = true) { + if (execution_mode != 'userscript') { + const res = await fetch(url); + const reader = res.body; + const stream = reader?.getReader(); + while (!stream?.closed) { + const buff = await stream?.read(); + if (buff?.done) { + break; + } + if (buff?.value) { + const e = (yield buff.value) as boolean; + if (e) { + stream?.cancel(); + reader?.cancel(); + break; + } + } + } + stream?.releaseLock(); + return; + } + const headers = await getHeaders(url); + const size = +headers['content-length']; + let ptr = 0; + let fetchSize = chunkSize; + while (ptr != size) { + //console.log('doing a fetch of ', url, ptr, ptr + fetchSize - 1); + const res = await ifetch(url, { headers: { range: `bytes=${ptr}-${ptr + fetchSize - 1}` } }) as any as Tampermonkey.Response; + const obj = headerStringToObject(res.responseHeaders); + if (!('content-length' in obj)) { + console.warn("no content lenght???", url); + break; + } const len = +obj['content-length']; + ptr += len; + if (fetchRestOnNonCanceled) + fetchSize = size; + const val = Buffer.from(await (res as any).arrayBuffer()); + const e = (yield val) as boolean; + //console.log('yeieledd, a', e); + if (e) { + break; + } + } +} diff --git a/src/pngv3.ts b/src/pngv3.ts index 5db7a55..8c48523 100644 --- a/src/pngv3.ts +++ b/src/pngv3.ts @@ -2,7 +2,6 @@ import { Buffer } from "buffer"; import type { ImageProcessor } from "./main"; import { PNGDecoder, PNGEncoder } from "./png"; import { buildPeeFile, decodeCoom3Payload, fireNotification, uploadFiles } from "./utils"; -import { GM_fetch } from "./requests"; const CUM3 = Buffer.from("doo\0" + "m"); diff --git a/src/pomf.ts b/src/pomf.ts index 9df30e7..7cbc040 100644 --- a/src/pomf.ts +++ b/src/pomf.ts @@ -1,8 +1,8 @@ import type { EmbeddedFile, ImageProcessor } from "./main"; -import { GM_fetch, GM_head } from "./requests"; import type { Buffer } from "buffer"; import thumbnail from "./assets/hasembed.png"; import { settings } from "./stores"; +import { getHeaders, ifetch, Platform } from "./platform"; const sources = [ { host: 'Catbox', prefix: 'files.catbox.moe/' }, @@ -52,7 +52,7 @@ const extract = async (b: Buffer, fn?: string) => { if (source && cs.prefix != source) continue; try { - await GM_head('https://' + cs.prefix + ext); + await getHeaders('https://' + cs.prefix + ext); rsource = 'https://' + cs.prefix + ext; break; } catch { @@ -64,7 +64,7 @@ const extract = async (b: Buffer, fn?: string) => { filename: ext, data: csettings.hotlink ? rsource! : async (lsn) => { try { - return (await GM_fetch(rsource, undefined, lsn)).arrayBuffer(); + return (await ifetch(rsource, undefined, lsn)).arrayBuffer(); } catch (e) { //404 } @@ -81,7 +81,7 @@ const has_embed = async (b: Buffer, fn?: string) => { if (source && cs.prefix != source) continue; try { - const e = await GM_head('https://' + cs.prefix + ext); + const e = await getHeaders('https://' + cs.prefix + ext); return true; } catch { // 404 diff --git a/src/thirdeye.ts b/src/thirdeye.ts index f8b1710..924b07e 100644 --- a/src/thirdeye.ts +++ b/src/thirdeye.ts @@ -1,9 +1,9 @@ import type { EmbeddedFile, ImageProcessor } from "./main"; -import { GM_fetch } from "./requests"; import { localLoad, settings } from "./stores"; import { Buffer } from "buffer"; import jpeg from 'jpeg-js'; import { bmvbhash_even } from "./phash"; +import { ifetch, Platform } from "./platform"; export let csettings: Parameters[0]; settings.subscribe(b => { @@ -129,7 +129,7 @@ const findFileFrom = async (b: Booru, hex: string, abort?: EventTarget) => { }*/ if (b.domain in cache && hex in cache[b.domain]) return cache[b.domain][hex] as BooruMatch[]; - const res = await GM_fetch(`https://${b.domain}${b.endpoint}${hex}`); + const res = await ifetch(`https://${b.domain}${b.endpoint}${hex}`); // might throw because some endpoint respond with invalid json when an error occurs const pres = await res.json(); const tran = b.quirks(pres).filter(e => !e.tags.some(e => black.has(e))); @@ -165,10 +165,10 @@ const extract = async (b: Buffer, fn?: string) => { url: result[0].page }, filename: fn!.substring(0, 33) + result[0].ext, - thumbnail: (await (await GM_fetch(prev || full)).arrayBuffer()), + thumbnail: csettings.hotlink ? (prev || full) : (await (await ifetch(prev || full)).arrayBuffer()), data: csettings.hotlink ? (full || prev) : (async (lsn) => { if (!cachedFile) - cachedFile = (await (await GM_fetch(full || prev, undefined, lsn)).arrayBuffer()); + cachedFile = (await (await ifetch(full || prev, undefined, lsn)).arrayBuffer()); return cachedFile; }) } as EmbeddedFile]; @@ -210,9 +210,9 @@ const has_embed = async (b: Buffer, fn?: string, prevlink?: string) => { if ((result && result.length != 0) && phashEn && prevlink) { const getHash = async (l: string) => { - const ogreq = await GM_fetch(l); + const ogreq = await ifetch(l); const origPreview = await ogreq.arrayBuffer(); - return await phash(Buffer.from(origPreview)); + return phash(Buffer.from(origPreview)); }; const [orighash, tehash] = await Promise.all([ getHash(prevlink), diff --git a/src/utils.ts b/src/utils.ts index a611dc8..b448f24 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,9 +1,9 @@ import { Buffer } from "buffer"; -import { GM_fetch, GM_head, headerStringToObject } from "./requests"; import thumbnail from "./assets/hasembed.png"; import type { EmbeddedFile } from './main'; import { settings } from "./stores"; import { filehosts } from "./filehosts"; +import { getHeaders, ifetch, Platform } from "./platform"; export let csettings: Parameters[0]; @@ -106,8 +106,12 @@ export const decodeCoom3Payload = async (buff: Buffer) => { return (await Promise.all(pees.map(async pee => { try { - const headers = headerStringToObject(await GM_head(pee)); - const res = await GM_fetch(pee, { + const m = pee.match(/(?https?):\/\/(?.*?)(?\/.*)/); + if (!m) + return; + const { domain, file } = m.groups!; + const headers = await getHeaders(pee); + const res = await ifetch(pee, { headers: { ranges: 'bytes=0-2048', 'user-agent': '' }, mode: 'cors', referrerPolicy: 'no-referrer', @@ -142,15 +146,24 @@ export const decodeCoom3Payload = async (buff: Buffer) => { if (hasThumbnail) { thumbsize = header.readInt32LE(ptr); ptr += 4; - thumb = Buffer.from(await (await GM_fetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${ptr + thumbsize}` } })).arrayBuffer()); + if (execution_mode == 'userscript') + thumb = Buffer.from(await (await ifetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${ptr + thumbsize}` } })).arrayBuffer()); + else + thumb = `https://loli.piss/${domain}${file}/${ptr}/${ptr + thumbsize}`; ptr += thumbsize; } const unzip = async (lsn?: EventTarget) => - Buffer.from(await (await GM_fetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${size - 1}` } }, lsn)).arrayBuffer()); + Buffer.from(await (await ifetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${size - 1}` } }, lsn)).arrayBuffer()); + let data; + if (execution_mode == 'userscript') { + data = size < 3072 ? await unzip() : unzip; + } else { + data = `https://loli.piss/${domain}${file}/${ptr}/${size - 1}`; + } return { filename: fn, // if file is small, then just get it fully - data: size < 3072 ? await unzip() : unzip, + data, thumbnail: thumb, } as EmbeddedFile; } catch (e) {