@ -1,12 +1,14 @@
import 'process' ; /// <reference lib="ES2021" />
/// <reference lib="dom" />
import { Buffer } from "buffer" ; import { Buffer } from "buffer" ;
import { appState , settings , initial_settings } from "./stores" ; import { appState , settings , initial_settings } from "./stores" ;
import { debounce } from './debounce' ; import { debounce } from './debounce' ;
import globalCss from './global.css' ; import globalCss from './global.css' ;
import pngv3 from "./pngv3" ; import pngv3 from "./pngv3" ;
import webm from "./webm" ; //import webm from "./webm";
import gif from "./gif" ; //import gif from "./gif";
import jpg from "./jpg" ; import jpg from "./jpg" ;
import thirdeye from "./thirdeye" ; import thirdeye from "./thirdeye" ;
import pomf from "./pomf" ; import pomf from "./pomf" ;
@ -21,27 +23,26 @@ import NotificationsHandler from './Components/NotificationsHandler.svelte';
import { fireNotification , getEmbedsFromCache , getSelectedFile } from "./utils" ; import { fireNotification , getEmbedsFromCache , getSelectedFile } from "./utils" ;
import { getQueryProcessor , QueryProcessor } from "./websites" ; import { getQueryProcessor , QueryProcessor } from "./websites" ;
import { ifetch , Platform , streamRemot e , supportedAltDomain , supportedMainDomain } from "./platform" ; import { ifetch , Platform , sendCmd , lqueu e , supportedAltDomain , supportedMainDomain } from "./platform" ;
import TextEmbeddingsSvelte from "./Components/TextEmbeddings.svelte" ; import TextEmbeddingsSvelte from "./Components/TextEmbeddings.svelte" ;
import { HydrusClient } from "./hydrus" ; import { HydrusClient } from "./hydrus" ;
import { registerPlugin } from 'linkifyjs' ; import { registerPlugin } from 'linkifyjs' ;
import ViewCountSvelte from "./Components/ViewCount.svelte" ; import ViewCountSvelte from "./Components/ViewCount.svelte" ;
import type { ImageProcessor , WorkerEmbeddedFile } from './processor.worker' ;
import ProcessWorkerAny from './processor.worker' ;
import { headerStringToObject } from "./requests" ;
const ProcessWorker = ProcessWorkerAny as ( ) = > Worker ;
if ( ! supportedMainDomain ( location . host ) && ! supportedAltDomain ( location . host ) ) if ( ! supportedMainDomain ( location . host ) && ! supportedAltDomain ( location . host ) )
throw "PEE not supported here, skipping" ; throw "PEE not supported here, skipping" ;
export interface ImageProcessor {
skip? : true ;
match ( fn : string ) : boolean ;
has_embed ( b : Buffer , fn? : string , prevurl? : string ) : boolean | string | undefined | Promise < boolean | string | undefined > ;
extract ( b : Buffer , fn? : string ) : EmbeddedFile [ ] | Promise < EmbeddedFile [ ] > ;
inject ? ( b : File , c : string [ ] ) : Buffer | Promise < Buffer > ;
}
let qp : QueryProcessor ; let qp : QueryProcessor ;
export let csettings : Parameters < typeof settings [ ' set ' ] > [ 0 ] ; export let csettings : Parameters < typeof settings [ ' set ' ] > [ 0 ] ;
le tprocessors : ImageProcessor [ ] = const processors : ImageProcessor [ ] =
[ thirdeye , pomf , pngv3 , jpg , webm , gif ] ; [ thirdeye , pomf , pngv3 , jpg ] ; //, webm, gif
let cappState : Parameters < typeof appState [ ' set ' ] > [ 0 ] ; let cappState : Parameters < typeof appState [ ' set ' ] > [ 0 ] ;
settings . subscribe ( async b = > { settings . subscribe ( async b = > {
@ -64,9 +65,9 @@ settings.subscribe(async b => {
} }
} }
csettings = b ; csettings = b ;
processors = [ . . . ( ! csettings . te ? [ thirdeye ] : [ ] ) , //processors = [...(!csettings.te ? [thirdeye] : []),
pngv3 , pomf , jpg , webm , gif // pngv3, pomf, jpg, webm, gif
] ; //];
} ) ; } ) ;
@ -79,7 +80,7 @@ type EmbeddedFileWithPreview = {
source? : string ; // can be like a twitter post this was posted in originally
source? : string ; // can be like a twitter post this was posted in originally
thumbnail : string | Buffer ; thumbnail : string | Buffer ;
filename : string ; filename : string ;
data : EmbeddedFileWithoutPreview [ 'data' ] | ( ( lisn? : EventTarget ) = > Promise < Buffer > ) ; data : EmbeddedFileWithoutPreview [ 'data' ] | ( ( lisn? : EventTarget ) = > Promise < Uint8Array > ) ;
} ; } ;
type EmbeddedFileWithoutPreview = { type EmbeddedFileWithoutPreview = {
@ -92,6 +93,7 @@ type EmbeddedFileWithoutPreview = {
export type EmbeddedFile = EmbeddedFileWithPreview | EmbeddedFileWithoutPreview ; export type EmbeddedFile = EmbeddedFileWithPreview | EmbeddedFileWithoutPreview ;
/ *
const processImage = async ( srcs : AsyncGenerator < string , void , void > , fn : string , hex : string , prevurl : string ) = > { const processImage = async ( srcs : AsyncGenerator < string , void , void > , fn : string , hex : string , prevurl : string ) = > {
const ret = await Promise . all ( processors . filter ( e = > e . match ( fn ) ) . map ( async proc = > { const ret = await Promise . all ( processors . filter ( e = > e . match ( fn ) ) . map ( async proc = > {
if ( proc . skip ) { if ( proc . skip ) {
@ -109,7 +111,7 @@ const processImage = async (srcs: AsyncGenerator<string, void, void>, fn: string
try { try {
const n = await srcs . next ( ) ; const n = await srcs . next ( ) ;
if ( n . done ) if ( n . done )
return ; // no more links to try
return ; // no more links to try
const iter = streamRemote ( n . value ) ; const iter = streamRemote ( n . value ) ;
if ( ! iter ) if ( ! iter )
return ; return ;
@ -129,7 +131,7 @@ const processImage = async (srcs: AsyncGenerator<string, void, void>, fn: string
} }
found = v ; found = v ;
} }
} while ( found !== false && ! chunk . done /* Because we only embed links now, it's safe to assume we get everything we need in the first chunk */ ) ; } while ( found !== false && ! chunk . done ) ;
succ = true ; succ = true ;
await iter . next ( true ) ; await iter . next ( true ) ;
if ( found !== true ) { if ( found !== true ) {
@ -143,7 +145,7 @@ const processImage = async (srcs: AsyncGenerator<string, void, void>, fn: string
} while ( ! succ ) ; } while ( ! succ ) ;
} ) ) ; } ) ) ;
return ret . filter ( e = > e ) . map ( e = > e ! ) ; return ret . filter ( e = > e ) . map ( e = > e ! ) ;
} ; } ; * /
const textToElement = < T = HTMLElement > ( s : string ) = > const textToElement = < T = HTMLElement > ( s : string ) = >
document . createRange ( ) . createContextualFragment ( s ) . children [ 0 ] as any as T ; document . createRange ( ) . createContextualFragment ( s ) . children [ 0 ] as any as T ;
@ -163,6 +165,8 @@ const buildCumFun = <T extends any[], U>(f: (args: T[]) => void, ...r: Parameter
} ; } ;
let pendingPosts : { id : number , op : number } [ ] = [ ] ; let pendingPosts : { id : number , op : number } [ ] = [ ] ;
let pendingNoPosts : { id : number , op : number } [ ] = [ ] ;
// should be equivalent to buildCumFun(signalNewEmbeds, 5000, {trailing: true})
// should be equivalent to buildCumFun(signalNewEmbeds, 5000, {trailing: true})
const signalNewEmbeds = debounce ( async ( ) = > { const signalNewEmbeds = debounce ( async ( ) = > {
// ensure user explicitely enabled telemetry
// ensure user explicitely enabled telemetry
@ -174,17 +178,22 @@ const signalNewEmbeds = debounce(async () => {
const boardname = location . pathname . match ( /\/([^/]*)\// ) ! [ 1 ] ; const boardname = location . pathname . match ( /\/([^/]*)\// ) ! [ 1 ] ;
// restructure to minimize redundancy
// restructure to minimize redundancy
const reshaped = Object . fromEntries ( [ . . . new Set ( pendingPosts . map ( e = > e . op ) ) ] . map ( e = > [ e , pendingPosts . filter ( p = > p . op == e ) . map ( e = > e . id ) ] ) ) ; const reshaped = Object . fromEntries ( [ . . . new Set ( pendingPosts . map ( e = > e . op ) ) ] . map ( e = > [ e , pendingPosts . filter ( p = > p . op == e ) . map ( e = > e . id ) ] ) ) ;
console . log ( reshaped ) ; const reshaped2 = Object . fromEntries ( [ . . . new Set ( pendingNoPosts . map ( e = > e . op ) ) ] . map ( e = > [ e , pendingPosts . filter ( p = > p . op == e ) . map ( e = > e . id ) ] ) ) ;
//console.log(reshaped);
const res = await ifetch ( "https://shoujo.coom.tech/listing/" + boardname , { const res = await ifetch ( "https://shoujo.coom.tech/listing/" + boardname , {
method : "POST" , method : "POST" ,
body : JSON.stringify ( reshaped ) , body : JSON.stringify ( {
emb : reshaped ,
noemb : reshaped2
} ) ,
headers : { headers : {
'content-type' : 'application/json' 'content-type' : 'application/json'
} }
} ) ; } ) ;
await res . json ( ) ; await res . json ( ) ;
pendingPosts = [ ] ; pendingPosts = [ ] ;
pendingNoPosts = [ ] ;
} catch ( e ) { } catch ( e ) {
// silently fail
// silently fail
console . error ( e ) ; console . error ( e ) ;
@ -201,6 +210,197 @@ const shouldUseCache = () => {
: location . hostname . includes ( 'b4k' ) ; : location . hostname . includes ( 'b4k' ) ;
} ; } ;
let cp : CommandProcessor ;
class CommandProcessor {
processor = ProcessWorker ( ) ;
genid = 0 ;
pendinggens : Record < number , AsyncGenerator > = { } ;
cmdid = 0 ;
pendingprom : Record < number , ( v ? : any ) = > void > = { } ;
constructor ( ) {
this . processor . onmessage = async ( msg ) = > {
let gen : AsyncGenerator ;
let res : IteratorResult < any , any > ;
switch ( msg . data . type ) {
case 'ipc' :
{
const id = msg . data . msg . id ;
if ( execution_mode != "userscript" ) {
if ( msg . data . msg . name == 'corsFetch' ) {
sendCmd ( msg . data . msg , msg . data . tr ) ;
lqueue [ id ] = ( res : any ) = > {
this . processor . postMessage ( {
type : 'ipc' ,
id ,
res
} ) ;
} ;
} else {
// for complitude, but technically the webworker doesn't run anything besides corsFetch
const repl : any = await sendCmd ( msg . data . msg , msg . data . tr ) ;
repl . id = id ;
this . processor . postMessage ( {
type : 'ipc' ,
id ,
res : repl
} ) ;
}
} else {
if ( msg . data . msg . name == 'fullyRead' ) {
// ignore
this . processor . postMessage ( {
type : 'ipc' ,
res : {
id ,
ok : 1
}
} ) ;
}
if ( msg . data . msg . name == 'corsFetch' ) {
const { args } = msg . data . msg ;
const res = await ifetch ( args [ 0 ] , args [ 1 ] ) ;
// don't report progress because monkeys don't have a way to expose partial responses anyway
const headersStr = ( res as any ) . responseHeaders ;
const headerObj = headerStringToObject ( headersStr ) ;
this . processor . postMessage ( {
type : 'ipc' ,
id ,
res : {
id ,
ok : res.ok || true ,
setRes : true ,
headers : headerObj ,
responseHeaders : headersStr ,
redirected : res.redirected ,
type : res . type ,
url : res.url ,
status : res.status ,
bodyUsed : res.bodyUsed ,
statusText : res.statusText ,
}
} ) ;
if ( ! args [ 1 ] . method || [ 'GET' , 'POST' ] . includes ( args [ 1 ] . method ) ) {
const data = await res . arrayBuffer ( ) ;
this . processor . postMessage ( {
type : 'ipc' ,
id ,
res : {
id ,
pushData : {
data
}
}
} , [ data ] ) ;
}
// let's hope these are delivered in order :%)
this . processor . postMessage ( {
type : 'ipc' ,
id ,
res : {
id ,
pushData : {
}
}
} , [ ] ) ;
}
// ignore other commands
}
} break ;
case 'reply' :
if ( msg . data . id in this . pendingprom ) {
this . pendingprom [ msg . data . id ] ( msg . data . res ) ;
delete this . pendingprom [ msg . data . id ] ;
}
break ;
case 'ag' :
gen = this . pendinggens [ msg . data . id ] ;
res = await gen . next ( msg . data . args ) ;
if ( res . done ) {
delete this . pendinggens [ msg . data . id ] ;
}
this . processor . postMessage ( {
type : 'ag' ,
id : msg.data.id ,
res
} ) ;
break ;
}
} ;
}
serializeArg ( m : any ) {
if ( m [ Symbol . toStringTag ] == 'AsyncGenerator' ) {
const genid = this . genid ++ ;
this . pendinggens [ genid ] = m ;
return {
type : 'AsyncGenerator' ,
id : genid
} ;
}
return m ;
}
sendCmd ( cmd : string , . . . args : any [ ] ) {
const id = this . cmdid ++ ;
this . processor . postMessage ( {
type : 'cmd' ,
id ,
fun : cmd ,
args : args.map ( a = > this . serializeArg ( a ) )
} ) ;
return new Promise < any > ( res = > {
this . pendingprom [ id ] = res ;
} ) ;
}
sendAg ( id : number , res : IteratorResult < any , any > ) {
this . processor . postMessage ( {
type : 'ag' ,
id , res // todo: call serializeArg?
} ) ;
}
processImage ( origlink : AsyncGenerator < string > , fn : string , md5 : string , thumb : string ) : Promise < [ WorkerEmbeddedFile [ ] , boolean ] [ ] > {
return this . sendCmd ( 'processImage' , origlink , fn , md5 , thumb ) ;
}
}
const convertToLocalEmbed = ( wef : WorkerEmbeddedFile ) = > {
let ret : EmbeddedFileWithPreview ;
ret = wef as any ;
// handles bigger files where data is represented as a {url, header} object
if ( typeof wef . data == "object" ) {
if ( ! ( wef . data instanceof Uint8Array ) ) {
const ref = wef . data ;
if ( ! wef . thumbnail )
return wef ;
ret = {
. . . wef ,
thumbnail : Buffer.from ( wef . thumbnail ) ,
data : async ( lsn ) = > {
return Buffer . from ( await ( await ifetch ( ref . url , { headers : ref.headers } , lsn ) ) . arrayBuffer ( ) ) ;
}
} ;
}
}
if ( wef . data instanceof Uint8Array ) {
ret . data = Buffer . from ( wef . data ) ;
}
if ( wef . thumbnail instanceof Uint8Array ) {
ret . thumbnail = Buffer . from ( wef . thumbnail ) ;
}
return ret ! ;
} ;
const processPost = async ( post : HTMLDivElement ) = > { const processPost = async ( post : HTMLDivElement ) = > {
const origlink = qp . getImageLink ( post ) ; const origlink = qp . getImageLink ( post ) ;
if ( ! origlink ) if ( ! origlink )
@ -208,17 +408,17 @@ const processPost = async (post: HTMLDivElement) => {
const thumbLink = qp . getThumbnailLink ( post ) ; const thumbLink = qp . getThumbnailLink ( post ) ;
if ( ! thumbLink ) if ( ! thumbLink )
return ; return ;
let res2 : [ EmbeddedFile [ ] , boolean ] [ ] | undefined = undefined ; let res2 : [ Worker EmbeddedFile[ ] , boolean ] [ ] | undefined = undefined ;
const op = + location . pathname . match ( /\/thread\/(.*)/ ) ! [ 1 ] ;
const reportEmbed = ( ) = > { const reportEmbed = ( ) = > {
if ( ! csettings ) if ( ! csettings )
return false ; return ;
if ( csettings . tm ) { if ( csettings . tm ) {
// dont report results from archive, only live threads
// dont report results from archive, only live threads
if ( [ 'boards.4chan.org' , 'boards.4channel.org' ] . includes ( location . host ) ) { if ( [ 'boards.4chan.org' , 'boards.4channel.org' ] . includes ( location . host ) ) {
if ( ! cappState . isCatalog ) { // only save from within threads
if ( ! cappState . isCatalog ) { // only save from within threads
// we must be in a thread, thus the following is valid
// we must be in a thread, thus the following is valid
const op = + location . pathname . match ( /\/thread\/(.*)/ ) ! [ 1 ] ;
pendingPosts . push ( { id : + ( post . id . match ( /([0-9]+)/ ) ! [ 1 ] ) , op } ) ; pendingPosts . push ( { id : + ( post . id . match ( /([0-9]+)/ ) ! [ 1 ] ) , op } ) ;
signalNewEmbeds ( ) ; // let it run async
signalNewEmbeds ( ) ; // let it run async
} }
@ -226,12 +426,29 @@ const processPost = async (post: HTMLDivElement) => {
} }
} ; } ;
const reportNoEmbed = ( ) = > {
if ( ! csettings )
return ;
if ( csettings . tm ) {
// dont report results from archive, only live threads
if ( [ 'boards.4chan.org' , 'boards.4channel.org' ] . includes ( location . host ) ) {
if ( ! cappState . isCatalog ) { // only save from within threads
// we must be in a thread, thus the following is valid
pendingNoPosts . push ( { id : + ( post . id . match ( /([0-9]+)/ ) ! [ 1 ] ) , op } ) ;
signalNewEmbeds ( ) ; // let it run async
}
}
}
} ;
try { try {
if ( shouldUseCache ( ) ) { if ( shouldUseCache ( ) ) {
res2 = await getEmbedsFromCache ( qp . getCurrentBoard ( ) , + qp . getCurrentThread ( ) ! , post . id ) ; res2 = await getEmbedsFromCache ( qp . getCurrentBoard ( ) , + qp . getCurrentThread ( ) ! , post . id ) ;
} }
if ( ! res2 ) { if ( ! res2 ) {
res2 = await processImage ( origlink , qp . getFilename ( post ) , qp . getMD5 ( post ) , thumbLink ) ; res2 = [ ] ;
const tmp = await cp . processImage ( origlink , qp . getFilename ( post ) , qp . getMD5 ( post ) , thumbLink ) ;
res2 . push ( . . . tmp ) ;
res2 = res2 ? . filter ( e = > e ) ; res2 = res2 ? . filter ( e = > e ) ;
} }
} catch ( e ) { } catch ( e ) {
@ -239,10 +456,10 @@ const processPost = async (post: HTMLDivElement) => {
return ; return ;
} }
if ( ! res2 || res2 . length == 0 ) if ( ! res2 || res2 . length == 0 )
return ; return reportNoEmbed ( ) ;
reportEmbed ( ) ; reportEmbed ( ) ;
post . querySelector ( '.post' ) ? . classList . add ( "embedfound" ) ; post . querySelector ( '.post' ) ? . classList . add ( "embedfound" ) ;
processAttachments ( post , res2 ? . flatMap ( e = > e ! [ 0 ] . map ( k = > [ k , e ! [ 1 ] ] as [ EmbeddedFile , boolean ] ) ) ) ; processAttachments ( post , res2 ? . flatMap ( e = > e ! [ 0 ] . map ( k = > [ convertToLocalEmbed ( k ) , e ! [ 1 ] ] as [ EmbeddedFile , boolean ] ) ) ) ;
} ; } ;
const versionCheck = async ( ) = > { const versionCheck = async ( ) = > {
@ -269,156 +486,118 @@ function copyTextToClipboard(text: string) {
} }
const scrapeBoard = async ( self : HTMLButtonElement ) = > { const scrapeBoard = async ( self : HTMLButtonElement ) = > {
if ( ! csettings ) / * i f ( ! c s e t t i n g s )
return false ; return false ;
if ( csettings . tm ) { if ( csettings . tm ) {
fireNotification ( "success" , "Scrapping board with telemetry on! Thank you for your service, selfless stranger ;_;7" ) ; fireNotification ( "success" , "Scrapping board with telemetry on! Thank you for your service, selfless stranger ;_;7" ) ;
} }
self . disabled = true ; self . disabled = true ;
self . textContent = "Searching..." ; self . textContent = "Searching..." ;
const boardname = location . pathname . match ( /\/([^/]*)\// ) ! [ 1 ] ; const boardname = location . pathname . match ( /\/([^/]*)\// ) ! [ 1 ] ;
const res = await ifetch ( ` https://a.4cdn.org/ ${ boardname } /threads.json ` ) ; const res = await ifetch ( ` https://a.4cdn.org/ ${ boardname } /threads.json ` ) ;
const pages = await res . json ( ) as Page [ ] ; const pages = await res . json ( ) as Page [ ] ;
type Page = { threads : Thread [ ] } type Page = { threads : Thread [ ] }
type Thread = { no : number ; posts : Post [ ] } ; type Thread = { no : number ; posts : Post [ ] } ;
type BasePost = { no : number , resto : number , tim : number } ; type BasePost = { no : number , resto : number , tim : number } ;
type PostWithFile = BasePost & { tim : number , ext : string , md5 : string , filename : string } ; type PostWithFile = BasePost & { tim : number , ext : string , md5 : string , filename : string } ;
type PostWithoutFile = BasePost & Record < string , unknown > ; type PostWithoutFile = BasePost & Record < string , unknown > ;
type Post = ( PostWithoutFile | PostWithFile ) ; type Post = ( PostWithoutFile | PostWithFile ) ;
fireNotification ( "info" , "Fetching all threads..." ) ; fireNotification ( "info" , "Fetching all threads..." ) ;
const threads = ( await Promise . all ( pages const threads = ( await Promise . all ( pages
. reduce ( ( a : Thread [ ] , b : Page ) = > [ . . . a , . . . b . threads ] , [ ] ) . reduce ( ( a : Thread [ ] , b : Page ) = > [ . . . a , . . . b . threads ] , [ ] )
. map ( e = > e . no ) . map ( e = > e . no )
. map ( async id = > { . map ( async id = > {
try { try {
const res = await ifetch ( ` https://a.4cdn.org/ ${ boardname } /thread/ ${ id } .json ` ) ; const res = await ifetch ( ` https://a.4cdn.org/ ${ boardname } /thread/ ${ id } .json ` ) ;
return await res . json ( ) as Thread ; return await res . json ( ) as Thread ;
} catch { } catch {
return undefined ; return undefined ;
}
} ) ) ) . filter ( e = > e ) . map ( e = > e as Thread ) ;
const filenames = threads
. reduce ( ( a , b ) = > [ . . . a , . . . b . posts . filter ( p = > p . ext )
. map ( p = > p as PostWithFile ) ] , [ ] as PostWithFile [ ] ) . filter ( p = > p . ext != '.webm' && p . ext != '.gif' )
. map ( p = > [ p . resto || p . no , ` https://i.4cdn.org/ ${ boardname } / ${ p . tim } ${ p . ext } ` , p . md5 , p . filename + p . ext , p . no ] as [ number , string , string , string , number ] ) ;
console . log ( filenames ) ;
fireNotification ( "info" , "Analyzing images..." ) ;
const n = 1 ;
//console.log(posts);
const processFile = ( src : string , fn : string , hex : string ) = > {
return Promise . all ( processors . filter ( e = > e . match ( fn ) ) . map ( async proc = > {
if ( proc . skip ) {
const md5 = Buffer . from ( hex , 'base64' ) ;
return await proc . has_embed ( md5 , fn ) ;
}
// TODO: Move this outside the loop?
const iter = streamRemote ( src ) ;
if ( ! iter )
return false ;
let cumul = Buffer . alloc ( 0 ) ;
let found : boolean | undefined ;
let chunk : ReadableStreamDefaultReadResult < Buffer > = { done : true } ;
do {
const { value , done } = await iter . next ( typeof found === "boolean" ) ;
if ( done ) {
chunk = { done : true } as ReadableStreamDefaultReadDoneResult ;
} else {
chunk = { done : false , value } as ReadableStreamDefaultReadValueResult < Buffer > ;
cumul = Buffer . concat ( [ cumul , value ! ] ) ;
const v = await proc . has_embed ( cumul ) ;
if ( typeof v == "string" ) {
return true ;
}
found = v ;
} }
} while ( found !== false && ! chunk . done ) ; } ) ) ) . filter ( e = > e ) . map ( e = > e as Thread ) ;
await iter . next ( true ) ; const filenames = threads
return found === true ; . reduce ( ( a , b ) = > [ . . . a , . . . b . posts . filter ( p = > p . ext )
} ) ) ; . map ( p = > p as PostWithFile ) ] , [ ] as PostWithFile [ ] ) . filter ( p = > p . ext != '.webm' && p . ext != '.gif' )
} ; . map ( p = > [ p . resto || p . no , ` https://i.4cdn.org/ ${ boardname } / ${ p . tim } ${ p . ext } ` , p . md5 , p . filename + p . ext , p . no ] as [ number , string , string , string , number ] ) ;
const range = ~ ~ ( filenames . length / n ) + 1 ;
const hasEmbed : typeof filenames = [ ] ; console . log ( filenames ) ;
const total = filenames . length ; fireNotification ( "info" , "Analyzing images..." ) ;
let processed = 0 ; const n = 1 ;
//console.log(posts);
const int = setInterval ( ( ) = > { const processFile = ( src : string , fn : string , hex : string ) = > {
fireNotification ( "info" , ` Processed [ ${ processed } / ${ total } ] files ` ) ; return Promise . all ( processors . filter ( e = > e . match ( fn ) ) . map ( async proc = > {
} , 5000 ) ; if ( proc . skip ) {
const md5 = Buffer . from ( hex , 'base64' ) ;
await Promise . all ( [ . . . new Array ( n + 1 ) ] . map ( async ( e , i ) = > { return await proc . has_embed ( md5 , fn ) ;
const postsslice = filenames . slice ( i * range , ( i + 1 ) * range ) ; }
for ( const post of postsslice ) { // TODO: Move this outside the loop?
try { const iter = streamRemote ( src ) ;
const res = await processFile ( post [ 1 ] , post [ 3 ] , post [ 2 ] ) ; if ( ! iter )
processed ++ ; return false ;
if ( res . some ( e = > e ) ) { let cumul = Buffer . alloc ( 0 ) ;
hasEmbed . push ( post ) ; let found : boolean | undefined ;
// dont report results from archive, only live threads
let chunk : ReadableStreamDefaultReadResult < Buffer > = { done : true } ;
if ( [ 'boards.4chan.org' , 'boards.4channel.org' ] . includes ( location . host ) ) { do {
pendingPosts . push ( { id : post [ 4 ] , op : post [ 0 ] } ) ; const { value , done } = await iter . next ( typeof found === "boolean" ) ;
signalNewEmbeds ( ) ; // let it run async
if ( done ) {
chunk = { done : true } as ReadableStreamDefaultReadDoneResult ;
} else {
chunk = { done : false , value } as ReadableStreamDefaultReadValueResult < Buffer > ;
cumul = Buffer . concat ( [ cumul , value ! ] ) ;
const v = await proc . has_embed ( cumul ) ;
if ( typeof v == "string" ) {
return true ;
}
found = v ;
}
} while ( found !== false && ! chunk . done ) ;
await iter . next ( true ) ;
return found === true ;
} ) ) ;
} ;
const range = ~ ~ ( filenames . length / n ) + 1 ;
const hasEmbed : typeof filenames = [ ] ;
const total = filenames . length ;
let processed = 0 ;
const int = setInterval ( ( ) = > {
fireNotification ( "info" , ` Processed [ ${ processed } / ${ total } ] files ` ) ;
} , 5000 ) ;
await Promise . all ( [ . . . new Array ( n + 1 ) ] . map ( async ( e , i ) = > {
const postsslice = filenames . slice ( i * range , ( i + 1 ) * range ) ;
for ( const post of postsslice ) {
try {
const res = await processFile ( post [ 1 ] , post [ 3 ] , post [ 2 ] ) ;
processed ++ ;
if ( res . some ( e = > e ) ) {
hasEmbed . push ( post ) ;
// dont report results from archive, only live threads
if ( [ 'boards.4chan.org' , 'boards.4channel.org' ] . includes ( location . host ) ) {
pendingPosts . push ( { id : post [ 4 ] , op : post [ 0 ] } ) ;
signalNewEmbeds ( ) ; // let it run async
}
} }
} catch ( e ) {
console . log ( e ) ;
} }
} catch ( e ) {
console . log ( e ) ;
} }
} } ) ) ;
} ) ) ;
clearInterval ( int ) ;
const counters : Record < number , number > = { } ;
for ( const k of hasEmbed )
counters [ k [ 0 ] ] = k [ 0 ] in counters ? counters [ k [ 0 ] ] + 1 : 1 ;
console . log ( counters ) ;
fireNotification ( "success" , "Processing finished! Results pasted in the clipboard" ) ;
const text = Object . entries ( counters ) . sort ( ( a , b ) = > b [ 1 ] - a [ 1 ] ) . map ( e = > ` >> ${ e [ 0 ] } ( ${ e [ 1 ] } ) ` ) . join ( '\n' ) ;
console . log ( text ) ;
copyTextToClipboard ( text ) ;
self . textContent = "Copy Results" ;
self . disabled = false ;
self . onclick = ( ) = > {
copyTextToClipboard ( text ) ;
} ;
} ;
const __DOMParser = execution_mode == "userscript" ? _DOMParser : DOMParser ;
const cleanupHTML = ( ndom : Document ) = > {
const evalWhenReady : string [ ] = [ ] ;
const addFromSource = ( elem : HTMLElement , url : string ) = > {
const scr = document . createElement ( 'script' ) ;
scr . type = 'text/javascript' ;
scr . src = url ;
elem . append ( scr ) ;
} ;
const addFromCode = ( elem : HTMLElement , sr : string ) = > {
const scr = document . createElement ( 'script' ) ;
scr . type = 'text/javascript' ;
scr . innerText = sr ;
elem . append ( scr ) ;
} ;
const rm = ( e : any ) = > e . remove ( ) ;
[ . . . ndom . head . children ] . filter ( e = > e . tagName == "SCRIPT" ) . forEach ( rm ) ;
[ . . . ndom . body . children ] . filter ( e = > e . tagName == "SCRIPT" ) . forEach ( rm ) ;
addFromSource ( ndom . body , "https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js" ) ;
addFromSource ( ndom . head , "https://based.coom.tech/highlight.pack.js" ) ;
/ *
addFromCode ( ndom . body , ` document.documentElement.className = "";document.documentElement.lang = "en";document.documentElement.dataset.site = "arch.b4k.co"; ` ) ;
addFromCode ( ndom . body , ` hljs.configure({ tableReplace: ' ' }); $ ('pre,code').each(function(i, block) { hljs.highlightBlock(block); }); var backend_vars = {"user_name":false,"user_email":false,"user_pass":"9fOK4K8","site_url":"https://arch.b4k.co/","default_url":"https://arch.b4k.co/","archive_url":"https://arch.b4k.co/","system_url":"https://arch.b4k.co/","api_url":"https://arch.b4k.co/","cookie_domain":null,"cookie_prefix":"foolfuuka_a2e7d4_","selected_theme":"foolz/foolfuuka-theme-foolfuuka","csrf_token_key":"csrf_token","images":{"banned_image":"https://arch.b4k.co/foolfuuka/foolz/foolfuuka-theme-foolfuuka/assets-1.2.28/images/banned-image.png","banned_image_width":150,"banned_image_height":150,"missing_image":"https://arch.b4k.co/foolfuuka/foolz/foolfuuka-theme-foolfuuka/assets-1.2.28/images/missing-image.jpg","missing_image_width":150,"missing_image_height":150},"gettext":{"submit_state":"Submitting","thread_is_real_time":"This thread is being displayed in real time.","update_now":"Update now","ghost_mode":"This thread has entered ghost mode. Your reply will be marked as a ghost post and will only affect the ghost index."},"board_shortname":"v"}; ` ) ;
// head
//body
await addFromSource ( ndom . body , "https://based.coom.tech/bootstrap.min.js" ) ;
await addFromSource ( ndom . body , "https://based.coom.tech/plugins.js" ) ;
await addFromSource ( ndom . body , "https://based.coom.tech/board.js" ) ; clearInterval ( int ) ;
await addFromSource ( ndom . body , "https://based.coom.tech/fuuka.js" ) ;
await addFromSource ( ndom . body , "https://based.coom.tech/lazyload.js" ) ; const counters : Record < number , number > = { } ;
* / for ( const k of hasEmbed )
return [ ndom . documentElement . innerHTML , evalWhenReady ] as [ string , string [ ] ] ; counters [ k [ 0 ] ] = k [ 0 ] in counters ? counters [ k [ 0 ] ] + 1 : 1 ;
console . log ( counters ) ;
fireNotification ( "success" , "Processing finished! Results pasted in the clipboard" ) ;
const text = Object . entries ( counters ) . sort ( ( a , b ) = > b [ 1 ] - a [ 1 ] ) . map ( e = > ` >> ${ e [ 0 ] } ( ${ e [ 1 ] } ) ` ) . join ( '\n' ) ;
console . log ( text ) ;
copyTextToClipboard ( text ) ;
self . textContent = "Copy Results" ;
self . disabled = false ;
self . onclick = ( ) = > {
copyTextToClipboard ( text ) ;
} ; * /
} ; } ;
let gmo : MutationObserver ; let gmo : MutationObserver ;
@ -564,6 +743,13 @@ const startup = async (is4chanX = true) => {
await bodyInit ; await bodyInit ;
} }
try {
cp = new CommandProcessor ( ) ;
} catch {
alert ( "You may be using 4chanX\n\nGo to 4chanX's settings, Advanced > JS Whitelist and add 'blob:' without quotes to the list." ) ;
return ;
}
if ( ! is4chanX && location . host . startsWith ( 'boards.4chan' ) ) { if ( ! is4chanX && location . host . startsWith ( 'boards.4chan' ) ) {
const notificationHost = document . createElement ( 'span' ) ; const notificationHost = document . createElement ( 'span' ) ;
new NotificationsHandler ( { new NotificationsHandler ( {
@ -793,7 +979,7 @@ function processAttachments(post: HTMLDivElement, ress: [EmbeddedFile, boolean][
target : textInsertCursor , target : textInsertCursor ,
props : { props : {
files : ress.map ( e = > e [ 0 ] ) . filter ( e = > files : ress.map ( e = > e [ 0 ] ) . filter ( e = >
Buffer . isBuffer ( e . data ) && e . filename . endsWith ( '.txt' ) && e . filename . startsWith ( 'message' ) ( Buffer . isBuffer ( e . data ) || e . data instanceof Uint8Array ) && e . filename . endsWith ( '.txt' ) && e . filename . startsWith ( 'message' )
) )
} }
} ) ; } ) ;
coomdev
2 years ago