Browse Source

Optimize buffer passing and fix progress on FF

pull/46/head
coomdev 2 years ago
parent
commit
8c710aaaf0
  1. 1
      .gitignore
  2. BIN
      PEE-chrome.crx
  3. 54
      build-chrome.js
  4. 22
      build-ff.js
  5. BIN
      chrome/_metadata/generated_indexed_rulesets/_ruleset1
  6. 22
      chrome/b4k-csp.json
  7. 35
      chrome/manifest.json
  8. 3
      chrome/options.js
  9. 2
      chrome_update.xml
  10. 14
      firefox/manifest.json
  11. 2
      firefox_update.json
  12. 1
      main.d.ts
  13. 3
      src/Components/Embedding.svelte
  14. 125
      src/background.ts
  15. 6
      src/main.ts
  16. 57
      src/platform.ts
  17. 20
      src/requests.ts
  18. 30
      src/utils.ts
  19. 2
      src/websites/index.ts

1
.gitignore

@ -18,7 +18,6 @@ out
*.data
ext.js
core.js
fag
rollup.config.js
.vscode/settings.json
key.pem

BIN
PEE-chrome.crx

Binary file not shown.

54
build-chrome.js

@ -92,9 +92,57 @@ const manif = {
}
};
const manif3 = {
"manifest_version": 3,
"update_url": "https://git.coom.tech/fuckjannies/lolipiss/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/chrome_update.xml",
"name": "PngExtraEmbedder",
"description": "Discover embedded files on 4chan and archives!",
"version": "0." + rev,
"icons": {
"64": "1449696017588.png"
},
"permissions": [
"notifications",
"clipboardWrite",
"activeTab",
"declarativeNetRequestWithHostAccess",
"contextMenus",
],
host_permissions: domains,
//"host_permissions":["<all_urls>"],
"web_accessible_resources": [{
"resources": ["*.html", "*.js"],
"matches": ["<all_urls>"]
}],
"content_scripts": [
{
"matches": domains,
"css": [],
"run_at": "document_start",
"js": ["dist/main.js"],
}
],
"declarative_net_request": {
"rule_resources": [
{
id: 'rule1',
enabled: true,
path: 'b4k-csp.json'
}
]
},
//"background": {
// hope I won't need that polyfill...
//"service_worker": "dist/background.js"
// }
};
(async () => {
let res;
const lmanif = manif3;
res = await esbuild
.build({
entryPoints: ["src/main.ts"],
@ -104,6 +152,7 @@ const manif = {
define: {
global: 'window',
execution_mode: '"chrome_api"',
manifest: lmanif.version,
isBackground: 'false',
BUILD_VERSION: JSON.stringify([0, rev])
},
@ -133,6 +182,7 @@ const manif = {
define: {
global: 'window',
execution_mode: '"chrome_api"',
manifest: lmanif.version,
isBackground: 'true',
BUILD_VERSION: JSON.stringify([0, rev])
},
@ -142,8 +192,8 @@ const manif = {
console.log(res.metafile.inputs);
console.log(Object.entries(res.metafile.inputs).sort((a, b) => a[1].bytes - b[1].bytes).map(e => `${e[0]} -> ${e[1].bytes}`).join('\n'));
writeFileSync('./chrome/manifest.json', JSON.stringify(manif, null, 2));
copyFileSync("./1449696017588.png", "./chrome/1449696017588.png");
writeFileSync('./chrome/manifest.json', JSON.stringify(lmanif, null, 2));
copyFileSync("./logo.png", "./chrome/1449696017588.png");
const ext = await crx.load('./chrome');
const crxBuffer = await ext.pack();

22
build-ff.js

@ -81,14 +81,16 @@ const manif = {
"js": ["polyfill.min.js", "dist/main.js"],
}
],
"background": {
persistent: true,
"scripts": [
"polyfill.min.js",
"browser-polyfill.min.js",
"dist/background.js"
]
}
"web_accessible_resources": ["*.html", "*.js",],
// "background": {
// persistent: true,
// "scripts": [
// "polyfill.min.js",
// "browser-polyfill.min.js",
// "dist/background.js"
// ]
// }
};
(async () => {
@ -103,6 +105,7 @@ const manif = {
define: {
global: 'window',
execution_mode: '"ff_api"',
manifest: manif.version,
isBackground: 'false',
BUILD_VERSION: JSON.stringify([0, rev])
},
@ -132,6 +135,7 @@ const manif = {
define: {
global: 'window',
execution_mode: '"ff_api"',
manifest: manif.version,
isBackground: 'true',
BUILD_VERSION: JSON.stringify([0, rev])
},
@ -142,7 +146,7 @@ const manif = {
console.log(Object.entries(res.metafile.inputs).sort((a, b) => a[1].bytes - b[1].bytes).map(e => `${e[0]} -> ${e[1].bytes}`).join('\n'));
writeFileSync('./firefox/manifest.json', JSON.stringify(manif, null, 2));
copyFileSync("./1449696017588.png", "./firefox/1449696017588.png");
copyFileSync("./logo.png", "./chrome/1449696017588.png");
res = await webExt.cmd.build({
sourceDir: './firefox/',

BIN
chrome/_metadata/generated_indexed_rulesets/_ruleset1

Binary file not shown.

22
chrome/b4k-csp.json

@ -0,0 +1,22 @@
[{
"id": 1,
"priority": 1,
"action": {
"type": "modifyHeaders",
"responseHeaders": [
{
"header": "content-security-policy",
"operation": "remove"
}
]
},
"condition": {
"urlFilter": "abc",
"initiatorDomains": [
"arch.b4k.co"
],
"resourceTypes": [
"main_frame"
]
}
}]

35
chrome/manifest.json

@ -1,20 +1,20 @@
{
"manifest_version": 2,
"manifest_version": 3,
"update_url": "https://git.coom.tech/fuckjannies/lolipiss/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/chrome_update.xml",
"name": "PngExtraEmbedder",
"description": "Discover embedded files on 4chan and archives!",
"version": "0.211",
"version": "0.213",
"icons": {
"64": "1449696017588.png"
},
"permissions": [
"notifications",
"clipboardWrite",
"menus",
"activeTab",
"webRequest",
"webRequestBlocking",
"contextMenus",
"declarativeNetRequestWithHostAccess",
"contextMenus"
],
"host_permissions": [
"https://*.coom.tech/*",
"https://boards.4channel.org/*",
"https://boards.4chan.org/*",
@ -51,6 +51,17 @@
"https://*.lolibooru.moe/*",
"https://*.allthefallen.moe/*"
],
"web_accessible_resources": [
{
"resources": [
"*.html",
"*.js"
],
"matches": [
"<all_urls>"
]
}
],
"content_scripts": [
{
"matches": [
@ -97,11 +108,13 @@
]
}
],
"background": {
"persistent": true,
"scripts": [
"browser-polyfill.min.js",
"dist/background.js"
"declarative_net_request": {
"rule_resources": [
{
"id": "rule1",
"enabled": true,
"path": "b4k-csp.json"
}
]
}
}

3
chrome/options.js

@ -0,0 +1,3 @@
console.log('optiosn')
debugger;
console.log('optiosn')

2
chrome_update.xml

@ -1,6 +1,6 @@
<?xml version='1.0' encoding='UTF-8'?>
<gupdate xmlns='http://www.google.com/update2/response' protocol='2.0'>
<app appid='ilffidhdekahjldemialkgahicnajchb'>
<updatecheck codebase='https://git.coom.tech/coomdev/PEE/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/PEE-chrome.crx' version='0.211' prodversionmin='64.0.3242' />
<updatecheck codebase='https://git.coom.tech/coomdev/PEE/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/PEE-chrome.crx' version='0.213' prodversionmin='64.0.3242' />
</app>
</gupdate>

14
firefox/manifest.json

@ -7,7 +7,7 @@
},
"name": "PngExtraEmbedder",
"description": "Discover embedded files on 4chan and archives!",
"version": "0.211",
"version": "0.213",
"icons": {
"64": "1449696017588.png"
},
@ -102,12 +102,8 @@
]
}
],
"background": {
"persistent": true,
"scripts": [
"polyfill.min.js",
"browser-polyfill.min.js",
"dist/background.js"
]
}
"web_accessible_resources": [
"*.html",
"*.js"
]
}

2
firefox_update.json

@ -1 +1 @@
{"addons":{"{34ac4994-07f2-44d2-8599-682516a6c6a6}":{"updates":[{"version":"0.211","update_link":"https://git.coom.tech/fuckjannies/lolipiss/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/pee-firefox.zip"}]}}}
{"addons":{"{34ac4994-07f2-44d2-8599-682516a6c6a6}":{"updates":[{"version":"0.213","update_link":"https://git.coom.tech/fuckjannies/lolipiss/raw/branch/%E4%B8%AD%E5%87%BA%E3%81%97/pee-firefox.zip"}]}}}

1
main.d.ts

@ -60,4 +60,5 @@ declare const execution_mode: 'userscript' | 'chrome_api' | 'ff_api';
declare const isBackground: boolean;
declare const chrome: typeof browser;
declare const _DOMParser: typeof DOMParser;
declare const manifest: 2 | 3;
declare function GM_addElement(parent: HTMLElement, tagname: string, attrs: Record<string, string>)

3
src/Components/Embedding.svelte

@ -6,6 +6,7 @@
import { createEventDispatcher } from "svelte";
import { Buffer } from "buffer";
import { getHeaders, Platform } from "../platform";
import { peeTarget } from "../utils";
export const dispatch = createEventDispatcher();
@ -111,7 +112,7 @@
let type: FileTypeResult | undefined;
if (typeof file.data != "string") {
unzipping = true;
let lisn = new EventTarget();
let lisn = new peeTarget();
lisn.addEventListener("progress", (e: any) => {
progress = e.detail;
});

125
src/background.ts

@ -21,6 +21,7 @@ const types = [
"websocket",
"xmlhttprequest"
] as browser.webRequest.ResourceType[];
const filts = {
urls: ["https://boards.4channel.org/*",
"https://boards.4chan.org/*",
@ -35,72 +36,23 @@ const filts = {
"https://de.catbox.moe/*",
"https://based.coom.tech/*",
"https://archiveofsins.com/*"],
types
types: ["main_frame", "sub_frame", "csp_report", "object", "other", "ping"] as browser.webRequest.ResourceType[]
};
/*
obj.webRequest.onBeforeSendHeaders.addListener(
function (details: any) {
details.requestHeaders.push({ name: 'x-basldas', value: 'aslkhfqe' });
return { requestHeaders: details.requestHeaders };
},
{ urls: ["<all_urls>"], types },
["blocking", "requestHeaders"]
);
*/
//const ridh = {
//} as any;
//obj.webRequest.onCompleted.addListener(details => {
// if (details.requestId in ridh)
// delete ridh[details.requestId];
//}, filts);
obj.webRequest.onHeadersReceived.addListener(details => {
if (details.url.startsWith('https://arch.b4k.co/') && details.type == "main_frame") {
const e = details.responseHeaders!.findIndex(e => e.name.toLowerCase() == "content-security-policy");
if (e >= 0)
details.responseHeaders![e].value = "";
return {
responseHeaders: [
...details.responseHeaders!,
{ name: 'access-control-allow-origin', value: '*' }
]
} as browser.webRequest.BlockingResponse;
}
}, filts, ['blocking', 'responseHeaders', ...(execution_mode == "chrome_api" ? ['extraHeaders' as 'blocking'] : [])]);
/*
obj.webRequest.onBeforeSendHeaders.addListener((details) => {
if (ridh[details.requestId]) {
const res = ridh[details.requestId];
return {
requestHeaders: [res]
} as browser.webRequest.BlockingResponse;
}
}, filts, ['blocking']);
obj.webRequest.onBeforeRequest.addListener((details) => {
const redirectUrl = details.url;
let idx: number;
if ((idx = redirectUrl.indexOf(spe)) == -1)
return;
const parts = redirectUrl.slice(idx + spe.length).split('/').filter(e => e);
const [domain, path, [start, end]] = [
parts[0],
parts.slice(1, -2).join('/'),
parts.slice(-2)
];
ridh[details.requestId] = {
name: "range",
value: `bytes=${start}-${end}`
};
return {
redirectUrl: `https://${domain}/${path}`,
requestHeaders: [{ name: 'x-akjflkd', value: 'qofh3r3' }]
} as browser.webRequest.BlockingResponse;
}, filts, ['blocking']);
*/
if (manifest == 2)
obj.webRequest.onHeadersReceived.addListener(details => {
if (details.url.startsWith('https://arch.b4k.co/') && details.type == "main_frame") {
const e = details.responseHeaders!.findIndex(e => e.name.toLowerCase() == "content-security-policy");
if (e >= 0)
details.responseHeaders![e].value = "";
return {
responseHeaders: [
...details.responseHeaders!,
{ name: 'access-control-allow-origin', value: '*' }
]
} as browser.webRequest.BlockingResponse;
}
}, filts, ['blocking', 'responseHeaders', ...(execution_mode == "chrome_api" ? ['extraHeaders' as 'blocking'] : [])]);
async function deserialize(src: any): Promise<any> {
switch (src.cls) {
@ -134,16 +86,10 @@ async function deserialize(src: any): Promise<any> {
}
}
const pendingFetches = new Map<browser.runtime.Port, { [id in number]: { fetchFully: boolean } }>();
const pendingFetches = new Map<MessagePort, { [id in number]: { fetchFully: boolean } }>();
const bgCorsFetch = async (c: browser.runtime.Port, id: number, input: string, init?: RequestInit) => {
/*if (typeof init?.signal == "number") {
const id = init?.signal as any as number;
const ab = new AbortController();
init.signal = ab.signal;
}*/
if (input.startsWith('//')) // wtf fireshit??
const bgCorsFetch = async (c: MessagePort, id: number, input: string, init?: RequestInit) => {
if (input.startsWith('//')) // firefox??
input = 'https:' + input;
if (init?.body && execution_mode == "chrome_api")
init.body = await deserialize(init.body);
@ -187,16 +133,16 @@ const bgCorsFetch = async (c: browser.runtime.Port, id: number, input: string, i
ltotal += chunk.byteLength;
c.postMessage({ id, progress: [ltotal, ctotal] });
if (!pendingFetches.get(c)![id].fetchFully) {
const url = URL.createObjectURL(new Blob([chunk]));
c.postMessage({ id, s: s++, pushData: { data: url } });
//const url = new Blob([chunk]);
c.postMessage({ id, s: s++, pushData: { data: chunk } }, [chunk.buffer]);
} else {
buff.push(Buffer.from(chunk));
}
},
close() {
if (buff.length > 0) {
const url = URL.createObjectURL(new Blob([Buffer.concat(buff)]));
c.postMessage({ id, s: s++, pushData: { data: url } });
const chunk = Buffer.concat(buff);
c.postMessage({ id, s: s++, pushData: { data: chunk } }, [chunk.buffer]);
buff = [];
}
const obj = pendingFetches.get(c)!;
@ -217,9 +163,28 @@ const bgCorsFetch = async (c: browser.runtime.Port, id: number, input: string, i
reader?.releaseLock();
};
obj.runtime.onConnect.addListener((c) => {
c.onMessage.addListener(async obj => {
const meself = new URL(obj.runtime.getURL('')).origin;
const waitConnect = (cb: any) => {
window.addEventListener("message", (msg) => {
//if (msg.origin === meself) {
cb(msg.ports[0]);
//}
});
};
const onMessage = (c: MessagePort, cb: any) =>
c.onmessage = (e) => {
cb(e.data);
};
waitConnect((c: any) => {
onMessage(c, async (obj: any) => {
const { id, name, args, sid, fid, url } = obj as { url?: string, fid?: number, sid?: number, id: number, name: string, args: Parameters<typeof Platform[keyof Methods<typeof Platform>]> };
if (name == "keepAlive") {
console.log('im alive, tho?');
return;
}
if (name == "abortCorsFetch") {
//chrome.runtime.sendMessage({ name, sid });
return;

6
src/main.ts

@ -129,7 +129,6 @@ const processImage = async (srcs: AsyncGenerator<string, void, void>, fn: string
onfound();
return [await proc.extract(cumul), false] as [EmbeddedFile[], boolean];
} catch {
debugger;
// ignore error and retry with another link
}
} while (!succ);
@ -196,7 +195,6 @@ const processPost = async (post: HTMLDivElement) => {
const versionCheck = async () => {
const txt = (await (await ifetch("https://git.coom.tech/coomdev/PEE/raw/branch/%e4%b8%ad%e5%87%ba%e3%81%97/main.meta.js")).text());
debugger;
const [lmajor, lminor] = txt.split('\n')
.filter(e => e.includes("// @version"))[0].match(/.*version\s+(.*)/)![1].split('.')
.map(e => +e);
@ -370,8 +368,8 @@ let gmo: MutationObserver;
const earlystartup = async () => {
if (location.host == 'arch.b4k.co' && execution_mode == "userscript") {
if (!GM_getValue("warning_seen2", false)) {
alert(`Due to b4k's admin being a faggot, PEE will get you banned, so the userscript version is disabled here`);
alert("Use the WebExtension version of PEE if you want to use b4k. Cool new features will be coming to it, too!");
alert(`Due to b4k's policies being mean, PEE will get you banned, so the userscript version is disabled here`);
alert("Use the WebExtension version of PEE if you want to use b4k!"); // "Cool new features will be coming to it, too", then MV3 happened.
GM_setValue("warning_seen2", true);
return false;
}

57
src/platform.ts

@ -1,27 +1,47 @@
import { GM_fetch, GM_head, headerStringToObject } from './requests';
let port: browser.runtime.Port;
const lqueue = {} as any;
const { port1, port2 } = new MessageChannel();
console.log(execution_mode, isBackground);
if (execution_mode != 'userscript' && !isBackground) {
// It has to be a content script
port = (chrome || browser).runtime.connect();
port.onMessage.addListener((e: any) => {
lqueue[e.id](e);
const iframe = document.createElement('iframe');
iframe.style.display = 'none';
iframe.name = location.origin;
const iframeloaded = new Promise(_ => {
iframe.onload = _;
});
iframe.src = `${chrome.runtime.getURL('')}options.html`;
const meself = new URL(chrome.runtime.getURL('')).origin;
document.documentElement.appendChild(iframe);
iframeloaded.then(() => {
iframe.contentWindow?.postMessage('', '*', [port2]);
});
port1.onmessage = (ev) => {
lqueue[ev.data.id](ev.data);
};
}
let gid = 0;
const sendCmd = <V>(cmd: any) => {
const visit = (e: any, cb: (e: any) => true | undefined) => {
if (typeof e == "object") {
if (!cb(e)) // true if we don't want to visit deeper
for (const p in e)
visit(e[p], cb);
} else
cb(e);
};
const sendCmd = <V>(cmd: any, tr?: Transferable[]) => {
const prom = new Promise<V>(_ => {
const id = gid++;
lqueue[id] = (e: any) => {
_(e.res);
delete lqueue[id];
};
port.postMessage({ id, ...cmd });
port1.postMessage({ id, ...cmd }, tr || []);
});
return prom;
};
@ -63,12 +83,6 @@ export class Platform {
}
}
const extrBlob = async (url: string) => {
const ret = await (await fetch(url)).arrayBuffer();
await sendCmd({ name: 'revoke', url });
return new Uint8Array(ret);
};
async function serialize(src: any): Promise<any> {
if (src instanceof FormData) {
const value = [];
@ -108,16 +122,6 @@ async function serialize(src: any): Promise<any> {
export const corsFetch = async (input: string, init?: RequestInit, lsn?: EventTarget) => {
const id = gid++;
/* if (init) {
if (init.signal) {
const sid = gid++;
init.signal.addEventListener("abort", () => {
port.postMessage({ name: 'abortCorsFetch', sid });
});
(init as any).signal = sid as any;
}
}*/
if (init?.body) {
// Chrom* can't pass around FormData and File/Blobs between
// the content and bg scripts, so the data is passed through bloburls
@ -152,7 +156,8 @@ export const corsFetch = async (input: string, init?: RequestInit, lsn?: EventTa
// this is computed from the background script because the content script may
// request everything to be delivered in one chunk, defeating the purpose
if (e.progress) {
lsn?.dispatchEvent(new CustomEvent("progress", { detail: e.progress }));
if (lsn)
lsn.dispatchEvent(new CustomEvent("progress", { detail: e.progress }));
}
if (e.pushData) {
@ -173,10 +178,10 @@ export const corsFetch = async (input: string, init?: RequestInit, lsn?: EventTa
// then these must be equal
// this also means that cmdbuff must contain 0 or more ordered commands that must be processed
// afterward until discontinuity
const processCmd = async (e: any) => {
const processCmd = (e: any) => {
if (e.pushData.data) {
const data = await extrBlob(e.pushData.data);
const data = new Uint8Array(e.pushData.data);
if (gcontroller)
gcontroller.enqueue(data);
@ -252,7 +257,7 @@ export const corsFetch = async (input: string, init?: RequestInit, lsn?: EventTa
}
});
port.postMessage({
port1.postMessage({
id, name: 'corsFetch', args: [input, init]
});
});

20
src/requests.ts

@ -1,8 +1,11 @@
const xmlhttprequest = typeof GM_xmlhttpRequest != 'undefined' ?
GM_xmlhttpRequest :
(typeof GM != "undefined" ?
GM.xmlHttpRequest :
(window as any)['GM_xmlhttpRequest']);
let xmlhttprequest: typeof GM['xmlHttpRequest'];
if (!isBackground)
xmlhttprequest = typeof GM_xmlhttpRequest != 'undefined' ?
GM_xmlhttpRequest :
(typeof GM != "undefined" ?
GM.xmlHttpRequest :
(window as any)['GM_xmlhttpRequest']);
export const headerStringToObject = (s: string) =>
Object.fromEntries(s.split('\n').map(e => {
@ -31,10 +34,10 @@ export function GM_head(...[url, opt]: Parameters<typeof fetch>) {
});
}
export let GM_fetch = (...[url, opt, lisn]: [...Parameters<typeof fetch>, EventTarget?]) => {
export const GM_fetch = (...[url, opt, lisn]: [...Parameters<typeof fetch>, EventTarget?]) => {
function blobTo(to: string, blob: Blob) {
if (to == "arrayBuffer" && blob.arrayBuffer) {
const ret = blob.arrayBuffer(); // Fuck TM
const ret = blob.arrayBuffer(); // Heck TM
if (ret)
return ret;
}
@ -90,9 +93,6 @@ export let GM_fetch = (...[url, opt, lisn]: [...Parameters<typeof fetch>, EventT
});
};
if ((window as any)['pagemode'])
GM_fetch = fetch as any;
const makePoolable = <T extends any[], U>(fun: (...args: T) => Promise<U>, getPoolSize: () => number) => {
const pool = [];
let pending = 0;

30
src/utils.ts

@ -203,7 +203,7 @@ export const decodeCoom3Payload = async (buff: Buffer) => {
thumbnail: thumb,
} as EmbeddedFile;
} catch (e) {
// niggers trying to fuck with bad links
// meanies trying to heck with bad links
console.warn(e);
}
}))).filter(e => e);
@ -290,4 +290,32 @@ export async function getFileFromHydrus(client: HydrusClient,
] as [number, EmbeddedFile];
})
);
}
export class peeTarget {
targets = {} as { [k in string]: Array<(e: any) => any> };
addEventListener(ev: string, cb: (e: any) => any) {
this.targets[ev] = this.targets[ev] || [];
this.targets[ev].push(cb);
}
dispatchEvent(ev: CustomEvent) {
const evs = this.targets[ev.type];
if (evs)
for (const cb of evs)
cb(ev);
return true;
}
removeEventListener(ev: string, cb: any) {
const evs = this.targets[ev];
if (!evs) return;
for (let i = 0; i < evs.length; ++i) {
if (evs[i] == cb) {
evs.splice(i, 1);
return;
}
}
}
}

2
src/websites/index.ts

@ -68,7 +68,7 @@ export const FoolFuuka: QueryProcessor = {
catalogControlHost: () => document.getElementById("index-options") as HTMLDivElement,
getImageLink: async function *(post: HTMLElement) {
yield post.querySelector('a[rel]')?.getAttribute('href') || '';
if (location.host == "arch.b4k.co") { //get fucked
if (location.host == "arch.b4k.co") { //get hecked
const pid = post.id.match(/\d+/)![0];
const board = location.pathname.match(/\/(..?.?)\//)![1];
const res = await ifetch(`https://archive.wakarimasen.moe/_/api/chan/post/?board=${board}&num=${pid}`);

Loading…
Cancel
Save