Browse Source

WIP native browser extensions

pull/46/head
coomdev 2 years ago
parent
commit
305d7f6a9f
  1. 5
      build.js
  2. 6
      main.d.ts
  3. 15
      package-lock.json
  4. 3
      package.json
  5. 27
      src/Components/Embedding.svelte
  6. 34
      src/background.ts
  7. 6
      src/filehosts.ts
  8. 45
      src/main.ts
  9. 130
      src/platform.ts
  10. 1
      src/pngv3.ts
  11. 8
      src/pomf.ts
  12. 12
      src/thirdeye.ts
  13. 25
      src/utils.ts

5
build.js

@ -20,7 +20,10 @@ let rev = +res.stdout;
bundle: true,
outfile: "./dist/main.js",
define: {
global: 'window'
global: 'window',
execution_mode: JSON.stringify(process.argv[2] || 'userscript'),
isBackground: JSON.stringify('false'),
BUILD_VERSION: JSON.stringify([0, rev])
},
inject: ['./esbuild.inject.js'],
plugins: [

6
main.d.ts

@ -16,4 +16,8 @@ declare module 'blockhash' {
}, bits: number, method: number) => string;
}
declare const QR: any;
declare const QR: any;
declare const BUILD_VERSION: [number, number];
declare const execution_mode: 'userscript' | 'chrome_api' | 'ff_api';
declare const isBackground: boolean;
declare const chrome: typeof browser;

15
package-lock.json

@ -31,7 +31,8 @@
"svelte": "^3.44.3",
"svelte-check": "^2.2.11",
"svelte-preprocess": "^4.10.1",
"typescript": "^4.5.4"
"typescript": "^4.5.4",
"web-ext-types": "^3.2.1"
}
},
"node_modules/@babel/code-frame": {
@ -4658,6 +4659,12 @@
"extsprintf": "^1.2.0"
}
},
"node_modules/web-ext-types": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
"integrity": "sha512-oQZYDU3W8X867h8Jmt3129kRVKklz70db40Y6OzoTTuzOJpF/dB2KULJUf0txVPyUUXuyzV8GmT3nVvRHoG+Ew==",
"dev": true
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@ -8003,6 +8010,12 @@
"extsprintf": "^1.2.0"
}
},
"web-ext-types": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-ext-types/-/web-ext-types-3.2.1.tgz",
"integrity": "sha512-oQZYDU3W8X867h8Jmt3129kRVKklz70db40Y6OzoTTuzOJpF/dB2KULJUf0txVPyUUXuyzV8GmT3nVvRHoG+Ew==",
"dev": true
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",

3
package.json

@ -36,7 +36,8 @@
"svelte": "^3.44.3",
"svelte-check": "^2.2.11",
"svelte-preprocess": "^4.10.1",
"typescript": "^4.5.4"
"typescript": "^4.5.4",
"web-ext-types": "^3.2.1"
},
"browser": {
"node:buffer": "buffer",

27
src/Components/Embedding.svelte

@ -4,10 +4,8 @@
import { beforeUpdate, tick } from 'svelte'
import type { EmbeddedFile } from '../main'
import { createEventDispatcher } from 'svelte'
import { GM_head, headerStringToObject } from '../requests'
import { text } from 'svelte/internal'
import App from './App.svelte'
import { Buffer } from 'buffer'
import { getHeaders, Platform } from '../platform'
export const dispatch = createEventDispatcher()
@ -49,8 +47,9 @@
const thumb = file.thumbnail || file.data
let type: FileTypeResult | undefined
if (typeof thumb != 'string') {
type = await fileTypeFromBuffer(thumb)
if (typeof thumb != "string") {
let buff = Buffer.isBuffer(thumb) ? thumb : await thumb();
type = await fileTypeFromBuffer(buff)
if (
!type &&
file.filename.endsWith('.txt') &&
@ -58,12 +57,15 @@
) {
type = { ext: 'txt', mime: 'text/plain' } as any
}
content = new Blob([thumb], { type: type?.mime })
content = new Blob([buff], { type: type?.mime })
url = URL.createObjectURL(content)
if (!type) return
} else {
let head = headerStringToObject(await GM_head(thumb, undefined))
type = { ext: '' as any, mime: head['content-type'].split(';')[0].trim() as any }
let head = await getHeaders(thumb)
type = {
ext: '' as any,
mime: head['content-type'].split(';')[0].trim() as any,
}
}
ftype = type.mime
isVideo = type.mime.startsWith('video/')
@ -125,8 +127,11 @@
} else {
url = file.data
furl = file.data
let head = headerStringToObject(await GM_head(file.data, undefined))
type = { ext: '' as any, mime: head['content-type'].split(';')[0].trim() as any }
let head = await getHeaders(file.data)
type = {
ext: '' as any,
mime: head['content-type'].split(';')[0].trim() as any,
}
}
if (!type) return
isVideo = type.mime.startsWith('video/')
@ -191,7 +196,7 @@
ev.preventDefault()
if (isNotChrome) {
window.open(src, '_blank')
} else await GM.openInTab(src, { active: false, insert: true })
} else await Platform.openInTab(src, { active: false, insert: true })
}
}

34
src/background.ts

@ -0,0 +1,34 @@
import { Platform } from "./platform";
const obj = execution_mode == "chrome_api" ? chrome : browser;
type Methods<T> = {
// eslint-disable-next-line @typescript-eslint/ban-types
[k in Exclude<keyof T, 'prototype'>]: T[k] extends Function ? T[k] : never;
};
obj.webRequest.onBeforeRequest.addListener((details) => {
const redirectUrl = details.url;
if (!redirectUrl.startsWith("https://loli.piss/"))
return;
const m = redirectUrl.match(/https:\/\/loli.piss\/(?<domain>.*?)(?<path>\/.*)\/(?<start>.*)\/(?<end>.*)/);
if (!m)
return;
const { domain, path, start, end } = m.groups!;
return {
redirectUrl: `https://${domain}${path}`,
requestHeaders: [{
name: 'range',
value: `bytes=${start}-${end}`
}]
} as browser.webRequest.BlockingResponse;
}, { urls: ['*://loli.piss/*'] }, ['blocking']);
obj.runtime.onConnect.addListener((c) => {
c.onMessage.addListener(async obj => {
const { id, name, args } = obj as {id: number, name: keyof Methods<typeof Platform>, args: Parameters<typeof Platform[keyof Methods<typeof Platform>]>};
const res = await Platform[name](...args);
c.postMessage({
id, res
});
});
});

6
src/filehosts.ts

@ -1,4 +1,4 @@
import { GM_fetch } from "./requests";
import { ifetch } from "./platform";
function parseForm(data: object) {
const form = new FormData();
@ -14,7 +14,7 @@ export const lolisafe = (domain: string, serving = domain) => ({
domain,
serving,
async uploadFile(f: Blob) {
const resp = await GM_fetch(`https://${domain}/api/upload`, {
const resp = await ifetch(`https://${domain}/api/upload`, {
headers: {
accept: "application/json",
},
@ -33,7 +33,7 @@ export const catbox = (domain: string, serving: string) => ({
domain,
serving,
async uploadFile(inj: Blob) {
const resp = await GM_fetch(`https://${domain}/user/api.php`, {
const resp = await ifetch(`https://${domain}/user/api.php`, {
method: 'POST',
body: parseForm({
reqtype: 'fileupload',

45
src/main.ts

@ -9,8 +9,6 @@ import jpg, { convertToPng } from "./jpg";
import thirdeye from "./thirdeye";
import pomf from "./pomf";
import { GM_fetch, GM_head, headerStringToObject } from "./requests";
import App from "./Components/App.svelte";
import ScrollHighlighter from "./Components/ScrollHighlighter.svelte";
import PostOptions from "./Components/PostOptions.svelte";
@ -23,6 +21,7 @@ import { buildPeeFile, fireNotification } from "./utils";
import { fileTypeFromBuffer } from "file-type";
import { getQueryProcessor, QueryProcessor } from "./websites";
import { lolisafe } from "./filehosts";
import { ifetch, Platform, streamRemote, supportedAltDomain } from "./platform";
export interface ImageProcessor {
skip?: true;
@ -49,38 +48,10 @@ appState.subscribe(v => {
cappState = v;
});
// most pngs are encoded with 65k idat chunks
async function* streamRemote(url: string, chunkSize = 72 * 1024, fetchRestOnNonCanceled = true) {
const headers = await GM_head(url);
const h = headerStringToObject(headers);
const size = +h['content-length'];
let ptr = 0;
let fetchSize = chunkSize;
while (ptr != size) {
//console.log('doing a fetch of ', url, ptr, ptr + fetchSize - 1);
const res = await GM_fetch(url, { headers: { range: `bytes=${ptr}-${ptr + fetchSize - 1}` } }) as any as Tampermonkey.Response<any>;
const obj = headerStringToObject(res.responseHeaders);
if (!('content-length' in obj)) {
console.warn("no content lenght???", url);
break;
} const len = +obj['content-length'];
ptr += len;
if (fetchRestOnNonCanceled)
fetchSize = size;
const val = Buffer.from(await (res as any).arrayBuffer());
const e = (yield val) as boolean;
//console.log('yeieledd, a', e);
if (e) {
break;
}
}
//console.log("streaming ended, ", ptr, size);
}
type EmbeddedFileWithPreview = {
page?: { title: string, url: string }; // can be a booru page
source?: string; // can be like a twitter post this was posted in originally
thumbnail: Buffer;
thumbnail: string | Buffer;
filename: string;
data: EmbeddedFileWithoutPreview['data'] | ((lisn?: EventTarget) => Promise<Buffer>);
};
@ -88,7 +59,7 @@ type EmbeddedFileWithPreview = {
type EmbeddedFileWithoutPreview = {
page: undefined;
source: undefined;
thumbnail: undefined;
thumbnail?: string;
filename: string;
data: string | Buffer;
};
@ -156,12 +127,12 @@ const processPost = async (post: HTMLDivElement) => {
const versionCheck = async () => {
const [lmajor, lminor] =
(await (await GM_fetch("https://git.coom.tech/coomdev/PEE/raw/branch/%e4%b8%ad%e5%87%ba%e3%81%97/main.meta.js"))
(await (await ifetch("https://git.coom.tech/coomdev/PEE/raw/branch/%e4%b8%ad%e5%87%ba%e3%81%97/main.meta.js"))
.text())
.split('\n')
.filter(e => e.includes("// @version"))[0].match(/.*version\s+(.*)/)![1].split('.')
.map(e => +e);
const [major, minor] = GM.info.script.version.split('.').map(e => +e);
const [major, minor] = BUILD_VERSION;
if (major < lmajor || (major == lmajor && minor < lminor)) {
fireNotification("info", `Last PEE version is ${lmajor}.${lminor}, you're on ${major}.${minor}`);
}
@ -183,7 +154,7 @@ const scrapeBoard = async (self: HTMLButtonElement) => {
self.disabled = true;
self.textContent = "Searching...";
const boardname = location.pathname.match(/\/(.*)\//)![1];
const res = await GM_fetch(`https://a.4cdn.org/${boardname}/threads.json`);
const res = await ifetch(`https://a.4cdn.org/${boardname}/threads.json`);
const pages = await res.json() as Page[];
type Page = { threads: Thread[] }
type Thread = { no: number; posts: Post[] };
@ -197,7 +168,7 @@ const scrapeBoard = async (self: HTMLButtonElement) => {
.map(e => e.no)
.map(async id => {
try {
const res = await GM_fetch(`https://a.4cdn.org/${boardname}/thread/${id}.json`);
const res = await ifetch(`https://a.4cdn.org/${boardname}/thread/${id}.json`);
return await res.json() as Thread;
} catch {
return undefined;
@ -391,7 +362,7 @@ const startup = async (is4chanX = true) => {
document.addEventListener('4chanXInitFinished', () => startup(true));
document.addEventListener('4chanParsingDone', () => startup(false), { once: true });
if (GM.info.script.matches.slice(2).some(m => m.includes(location.host))) {
if (supportedAltDomain(location.host)) {
window.addEventListener('load', () => {
startup(false);
}, { once: true });

130
src/platform.ts

@ -0,0 +1,130 @@
import { Buffer } from 'ts-ebml/lib/tools';
import 'web-ext-types';
import { GM_fetch, GM_head, headerStringToObject } from './requests';
let port: browser.runtime.Port;
const lqueue: ((e: any) => boolean)[] = [];
if (execution_mode != 'userscript' && !isBackground) {
port = browser.runtime.connect();
port.onMessage.addListener((e: any) => {
const k = lqueue.map(f => f(e));
for (let i = k.length - 1; i != -1; --i) {
if (k[i])
lqueue.splice(i, 1);
}
});
}
let gid = 0;
const bridge = <U extends any[], V, T extends (...args: U) => V>(name: string, f: T) => {
if (execution_mode != 'userscript' && !isBackground)
return f;
return (...args: U) => {
const id = gid++;
const prom = new Promise<V>(_ => {
lqueue.push((e: any) => {
if (e.id != id)
return false;
_(e.res);
return true;
});
port.postMessage({
id, name, args
});
});
return prom;
};
};
// eslint-disable-next-line @typescript-eslint/ban-types
const Bridged = (ctor: any) => {
const keys = Object.getOwnPropertyNames(ctor).filter(k => typeof ctor[k] == "function");
for (const k of keys)
ctor[k] = bridge(k, ctor[k]);
};
export function supportedAltDomain(s: string) {
if (execution_mode == 'userscript')
return GM.info.script.matches.slice(2).some(m => m.includes(s));
return false;
}
// Used to call background-only APIs from content scripts
@Bridged
export class Platform {
static async openInTab(src: string, opts: { active: boolean, insert: boolean }) {
if (execution_mode == 'userscript') {
return GM.openInTab(src, opts);
}
const obj = execution_mode == "chrome_api" ? chrome : browser;
if (execution_mode == 'chrome_api') {
let i: number | undefined;
if (opts.insert)
i = (await obj.tabs.getCurrent()).index + 1;
return obj.tabs.create({ active: opts.active, url: src, index: i });
}
}
}
export async function getHeaders(s: string) {
if (execution_mode == 'userscript')
return headerStringToObject(await GM_head(s));
const res = await fetch(s, {
method: "HEAD"
});
return [...res.headers.entries()].reduce((a, b) => (a[b[0]] = b[1], a), {} as ReturnType<typeof headerStringToObject>);
}
export async function ifetch(...[url, opt, lisn]: [...Parameters<typeof fetch>, EventTarget?]): ReturnType<typeof fetch> {
if (execution_mode != "userscript")
return fetch(url, opt);
return GM_fetch(url, opt, lisn);
}
// most pngs are encoded with 65k idat chunks
export async function* streamRemote(url: string, chunkSize = 72 * 1024, fetchRestOnNonCanceled = true) {
if (execution_mode != 'userscript') {
const res = await fetch(url);
const reader = res.body;
const stream = reader?.getReader();
while (!stream?.closed) {
const buff = await stream?.read();
if (buff?.done) {
break;
}
if (buff?.value) {
const e = (yield buff.value) as boolean;
if (e) {
stream?.cancel();
reader?.cancel();
break;
}
}
}
stream?.releaseLock();
return;
}
const headers = await getHeaders(url);
const size = +headers['content-length'];
let ptr = 0;
let fetchSize = chunkSize;
while (ptr != size) {
//console.log('doing a fetch of ', url, ptr, ptr + fetchSize - 1);
const res = await ifetch(url, { headers: { range: `bytes=${ptr}-${ptr + fetchSize - 1}` } }) as any as Tampermonkey.Response<any>;
const obj = headerStringToObject(res.responseHeaders);
if (!('content-length' in obj)) {
console.warn("no content lenght???", url);
break;
} const len = +obj['content-length'];
ptr += len;
if (fetchRestOnNonCanceled)
fetchSize = size;
const val = Buffer.from(await (res as any).arrayBuffer());
const e = (yield val) as boolean;
//console.log('yeieledd, a', e);
if (e) {
break;
}
}
}

1
src/pngv3.ts

@ -2,7 +2,6 @@ import { Buffer } from "buffer";
import type { ImageProcessor } from "./main";
import { PNGDecoder, PNGEncoder } from "./png";
import { buildPeeFile, decodeCoom3Payload, fireNotification, uploadFiles } from "./utils";
import { GM_fetch } from "./requests";
const CUM3 = Buffer.from("doo\0" + "m");

8
src/pomf.ts

@ -1,8 +1,8 @@
import type { EmbeddedFile, ImageProcessor } from "./main";
import { GM_fetch, GM_head } from "./requests";
import type { Buffer } from "buffer";
import thumbnail from "./assets/hasembed.png";
import { settings } from "./stores";
import { getHeaders, ifetch, Platform } from "./platform";
const sources = [
{ host: 'Catbox', prefix: 'files.catbox.moe/' },
@ -52,7 +52,7 @@ const extract = async (b: Buffer, fn?: string) => {
if (source && cs.prefix != source)
continue;
try {
await GM_head('https://' + cs.prefix + ext);
await getHeaders('https://' + cs.prefix + ext);
rsource = 'https://' + cs.prefix + ext;
break;
} catch {
@ -64,7 +64,7 @@ const extract = async (b: Buffer, fn?: string) => {
filename: ext,
data: csettings.hotlink ? rsource! : async (lsn) => {
try {
return (await GM_fetch(rsource, undefined, lsn)).arrayBuffer();
return (await ifetch(rsource, undefined, lsn)).arrayBuffer();
} catch (e) {
//404
}
@ -81,7 +81,7 @@ const has_embed = async (b: Buffer, fn?: string) => {
if (source && cs.prefix != source)
continue;
try {
const e = await GM_head('https://' + cs.prefix + ext);
const e = await getHeaders('https://' + cs.prefix + ext);
return true;
} catch {
// 404

12
src/thirdeye.ts

@ -1,9 +1,9 @@
import type { EmbeddedFile, ImageProcessor } from "./main";
import { GM_fetch } from "./requests";
import { localLoad, settings } from "./stores";
import { Buffer } from "buffer";
import jpeg from 'jpeg-js';
import { bmvbhash_even } from "./phash";
import { ifetch, Platform } from "./platform";
export let csettings: Parameters<typeof settings['set']>[0];
settings.subscribe(b => {
@ -129,7 +129,7 @@ const findFileFrom = async (b: Booru, hex: string, abort?: EventTarget) => {
}*/
if (b.domain in cache && hex in cache[b.domain])
return cache[b.domain][hex] as BooruMatch[];
const res = await GM_fetch(`https://${b.domain}${b.endpoint}${hex}`);
const res = await ifetch(`https://${b.domain}${b.endpoint}${hex}`);
// might throw because some endpoint respond with invalid json when an error occurs
const pres = await res.json();
const tran = b.quirks(pres).filter(e => !e.tags.some(e => black.has(e)));
@ -165,10 +165,10 @@ const extract = async (b: Buffer, fn?: string) => {
url: result[0].page
},
filename: fn!.substring(0, 33) + result[0].ext,
thumbnail: (await (await GM_fetch(prev || full)).arrayBuffer()),
thumbnail: csettings.hotlink ? (prev || full) : (await (await ifetch(prev || full)).arrayBuffer()),
data: csettings.hotlink ? (full || prev) : (async (lsn) => {
if (!cachedFile)
cachedFile = (await (await GM_fetch(full || prev, undefined, lsn)).arrayBuffer());
cachedFile = (await (await ifetch(full || prev, undefined, lsn)).arrayBuffer());
return cachedFile;
})
} as EmbeddedFile];
@ -210,9 +210,9 @@ const has_embed = async (b: Buffer, fn?: string, prevlink?: string) => {
if ((result && result.length != 0) && phashEn && prevlink) {
const getHash = async (l: string) => {
const ogreq = await GM_fetch(l);
const ogreq = await ifetch(l);
const origPreview = await ogreq.arrayBuffer();
return await phash(Buffer.from(origPreview));
return phash(Buffer.from(origPreview));
};
const [orighash, tehash] = await Promise.all([
getHash(prevlink),

25
src/utils.ts

@ -1,9 +1,9 @@
import { Buffer } from "buffer";
import { GM_fetch, GM_head, headerStringToObject } from "./requests";
import thumbnail from "./assets/hasembed.png";
import type { EmbeddedFile } from './main';
import { settings } from "./stores";
import { filehosts } from "./filehosts";
import { getHeaders, ifetch, Platform } from "./platform";
export let csettings: Parameters<typeof settings['set']>[0];
@ -106,8 +106,12 @@ export const decodeCoom3Payload = async (buff: Buffer) => {
return (await Promise.all(pees.map(async pee => {
try {
const headers = headerStringToObject(await GM_head(pee));
const res = await GM_fetch(pee, {
const m = pee.match(/(?<protocol>https?):\/\/(?<domain>.*?)(?<file>\/.*)/);
if (!m)
return;
const { domain, file } = m.groups!;
const headers = await getHeaders(pee);
const res = await ifetch(pee, {
headers: { ranges: 'bytes=0-2048', 'user-agent': '' },
mode: 'cors',
referrerPolicy: 'no-referrer',
@ -142,15 +146,24 @@ export const decodeCoom3Payload = async (buff: Buffer) => {
if (hasThumbnail) {
thumbsize = header.readInt32LE(ptr);
ptr += 4;
thumb = Buffer.from(await (await GM_fetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${ptr + thumbsize}` } })).arrayBuffer());
if (execution_mode == 'userscript')
thumb = Buffer.from(await (await ifetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${ptr + thumbsize}` } })).arrayBuffer());
else
thumb = `https://loli.piss/${domain}${file}/${ptr}/${ptr + thumbsize}`;
ptr += thumbsize;
}
const unzip = async (lsn?: EventTarget) =>
Buffer.from(await (await GM_fetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${size - 1}` } }, lsn)).arrayBuffer());
Buffer.from(await (await ifetch(pee, { headers: { 'user-agent': '', range: `bytes=${ptr}-${size - 1}` } }, lsn)).arrayBuffer());
let data;
if (execution_mode == 'userscript') {
data = size < 3072 ? await unzip() : unzip;
} else {
data = `https://loli.piss/${domain}${file}/${ptr}/${size - 1}`;
}
return {
filename: fn,
// if file is small, then just get it fully
data: size < 3072 ? await unzip() : unzip,
data,
thumbnail: thumb,
} as EmbeddedFile;
} catch (e) {

Loading…
Cancel
Save