import * as tf from '@tensorflow/tfjs'; import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'; import modelJSON from './model.json'; const charset = [' ', '0', '2', '4', '8', 'A', 'D', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'R', 'S', 'T', 'V', 'W', 'X', 'Y']; let weightsData: Uint8Array; // base64 encoded weights let model: tf.LayersModel; tf.enableProdMode(); const wasmToUrl = (wasm: any) => { const blb = new Blob([wasm], { type: 'application/wasm' }); return URL.createObjectURL(blb); }; const backendloaded = (async () => { try { // dead code elimination should occur here // eslint-disable-next-line camelcase if (execution_mode === 'userscript' || execution_mode === 'test') { weightsData = (await import('./group1-shard1of1.bin')).default; const tfwasmthreadedsimd = (await import('./tfjs-backend-wasm-threaded-simd.wasm')).default; const tfwasmsimd = (await import('./tfjs-backend-wasm-simd.wasm')).default; const tfwasm = (await import('./tfjs-backend-wasm.wasm')).default; setWasmPaths({ 'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm), 'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd), 'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd) }); } else { weightsData = new Uint8Array(await (await fetch(chrome.runtime.getURL('./group1-shard1of1.bin'))).arrayBuffer()); const args = { 'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'), 'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'), 'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm') }; setWasmPaths(args); } const l = await tf.setBackend('wasm'); console.log('tf backend loaded', l); } catch (err) { console.log('tf err', err); } })(); function toggle(obj: HTMLElement, v?: any) { if (v) obj.style.display = ''; else obj.style.display = 'none'; } const iohander: tf.io.IOHandler = { load: function () { return new Promise((resolve, reject) => { resolve({ modelTopology: modelJSON.modelTopology, weightSpecs: modelJSON.weightsManifest[0].weights as tf.io.WeightsManifestEntry[], weightData: weightsData.buffer, format: modelJSON.format, generatedBy: modelJSON.generatedBy, convertedBy: modelJSON.convertedBy }); }); } }; async function load() { model = await tf.loadLayersModel(iohander); return model; } function black(x: number) { return x < 64; } // Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a // non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider. function calculateDisorder(imgdata: ImageData) { const a = imgdata.data; const w = imgdata.width; const h = imgdata.height; const pic: number[] = []; const visited: number[] = []; for (let c = 0; c < w * h; c++) { if (visited[c]) continue; if (!black(a[c * 4])) continue; let blackCount = 0; const items: number[] = []; const toVisit: number[] = [c]; while (toVisit.length > 0) { const cc = toVisit[toVisit.length - 1]; toVisit.splice(toVisit.length - 1, 1); if (visited[cc]) continue; visited[cc] = 1; if (black(a[cc * 4])) { items.push(cc); blackCount++; toVisit.push(cc + 1); toVisit.push(cc - 1); toVisit.push(cc + w); toVisit.push(cc - w); } } if (blackCount >= 24) { items.forEach(function (x) { pic[x] = 1; }); } } let res = 0; let total = 0; for (let c = 0; c < w * h - w; c++) { if (pic[c] !== pic[c + w]) res += 1; if (pic[c]) total += 1; } return res / (total === 0 ? 1 : total); } /* * decide if a pixel is closer to black than to white. * return 0 for white, 1 for black */ function pxlBlackOrWhite(r: number, g: number, b: number) { return (r + g + b > 384) ? 0 : 1; } /* * Get bordering pixels of transparent areas (the outline of the circles) * and return their coordinates with the neighboring color. */ function getBoundries(imgdata: ImageData) { const data = imgdata.data; const width = imgdata.width; let i = data.length - 1; let cl = 0; let cr = 0; const chkArray = []; let opq = true; while (i > 0) { // alpha channel above 128 is assumed opaque const a = data[i] > 128; if (a !== opq) { if ((data[i - 4] > 128) === opq) { // ignore just 1-width areas i -= 4; continue; } if (a) { /* transparent pixel to its right */ /* // set to color blue (for debugging) data[i + 4] = 255; data[i + 3] = 255; data[i + 2] = 0; data[i + 1] = 0; */ const pos = (i + 1) / 4; const x = pos % width; const y = (pos - x) / width; // 1: black, 0: white const clr = pxlBlackOrWhite(data[i - 1], data[i - 2], data[i - 3]); chkArray.push([x, y, clr]); cr += 1; } else { /* opaque pixel to its right */ /* // set to color red (for debugging) data[i] = 255; data[i - 1] = 0; data[i - 2] = 0; data[i - 3] = 255; */ const pos = (i - 3) / 4; const x = pos % width; const y = (pos - x) / width; // 1: black, 0: white const clr = pxlBlackOrWhite(data[i + 1], data[i + 2], data[i + 3]); chkArray.push([x, y, clr]); cl += 1; } opq = a; } i -= 4; } return chkArray; } /* * slide the background image and compare the colors of the border pixels in * chkArray, the position with the most matches wins * Return in slider-percentage. */ function getBestPos(bgdata: ImageData, chkArray: number[][], slideWidth: number) { const data = bgdata.data; const width = bgdata.width; let bestSimilarity = 0; let bestPos = 0; for (let s = 0; s <= slideWidth; s += 1) { let similarity = 0; const amount = chkArray.length; for (let p = 0; p < amount; p += 1) { const chk = chkArray[p]; const x = chk[0] + s; const y = chk[1]; const clr = chk[2]; const off = (y * width + x) * 4; const bgclr = pxlBlackOrWhite(data[off], data[off + 1], data[off + 2]); if (bgclr === clr) { similarity += 1; } } if (similarity > bestSimilarity) { bestSimilarity = similarity; bestPos = s; } } return bestPos / slideWidth * 100; } async function getImageDataFromURI(uri: string) { const image = await imageFromUri(uri); if (!image) throw new Error("No image"); const canvas = document.createElement('canvas'); canvas.width = image.width; canvas.height = image.height; const ctx = canvas.getContext('2d')!; ctx.drawImage(image, 0, 0); return ctx.getImageData(0, 0, canvas.width, canvas.height); } async function slideCaptcha(tfgElement: HTMLElement, tbgElement: HTMLElement, sliderElement: HTMLInputElement) { // get data uris for captcha back- and foreground const tbgUri = tbgElement.style.backgroundImage.slice(5, -2); const tfgUri = tfgElement.style.backgroundImage.slice(5, -2); // load foreground (image with holes) const igd = await getImageDataFromURI(tfgUri); // get array with pixels of foreground // that we compare to background const chkArray = getBoundries(igd); // load background (image that gets slid) const sigd = await getImageDataFromURI(tbgUri); const slideWidth = sigd.width - igd.width; // slide, compare and get best matching position const sliderPos = getBestPos(sigd, chkArray, slideWidth); // slide in the UI sliderElement.value = '' + sliderPos; (sliderElement as any).dispatchEvent(new Event('input'), { bubbles: true }); return 0 - (sliderPos / 2); } // returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50) async function imageFromCanvas(img: HTMLImageElement, bg: HTMLImageElement, off: number | null) { const h = img.height; const w = img.width; const th = 80; const ph = 0; const pw = 16; const scale = th / h; const canvas = document.createElement('canvas'); const fcanvas = document.createElement('canvas'); const cw = w * scale + pw * 2; canvas.width = cw >= 300 ? 300 : cw; canvas.height = th; fcanvas.width = 300; fcanvas.height = 80; const ctx = canvas.getContext('2d')!; const fctx = fcanvas.getContext('2d')!; // used to contain the captcha stretched to 300w ctx.fillStyle = 'rgb(238,238,238)'; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.translate(canvas.width / 2, canvas.height / 2); const draw = function (off: number) { if (bg) { const border = 4; ctx.drawImage( bg, -off + border, 0, w - border * 2, h, -w / 2 + border, -h / 2, w - border * 2, h ); } ctx.drawImage(img, -w / 2, -h / 2, w, h); fctx.drawImage(canvas, 0, 0, 300, 80); }; if (bg && off == null) { off = await slideCaptcha(document.getElementById('t-fg')!, document.getElementById('t-bg')!, document.getElementById('t-slider') as HTMLInputElement); } draw(off || 0); return fctx.getImageData(0, 0, 300, 80); } function toMonochromeFloat(px: Uint8ClampedArray) { const ret = Array(px.length >> 2); for (let i = 0; i < px.length; i += 4) { ret[i >> 2] = px[i] / 255; } return ret; } const greedyCTCDecode = (yPred: tf.Tensor) => tf.tidy(() => yPred.argMax(-1).arraySync()); function processCTCDecodedSequence(decodedSequence: number[], blankLabel = 0) { const result = []; let prevLabel = blankLabel; for (const label of decodedSequence) { if (label !== blankLabel && label !== prevLabel) { result.push(label); } prevLabel = label; } return result; } function indicesToSymbols(decodedIndices: number[]) { return decodedIndices.map(index => charset[index - 1] || ''); } async function predict(img: HTMLImageElement, bg: HTMLImageElement, off: number) { if (!model) { model = await load(); } const image = await imageFromCanvas(img, bg, off); if (!image) throw new Error("Failed to gen image"); const mono = toMonochromeFloat(image.data); const filtered2 = tf.tensor3d(mono, [image.height, image.width, 1]); const prediction = model.predict(filtered2.transpose([1, 0, 2]).expandDims(0)); let d: tf.TypedArray; if (Array.isArray(prediction)) throw new Error("Unexpected inference type"); const v = greedyCTCDecode(prediction) as number[][]; const s = processCTCDecodedSequence(v[0], charset.length + 1); return indicesToSymbols(s).join('').trim(); } async function imageFromUri(uri: string) { if (uri.startsWith('url("')) { uri = uri.substr(5, uri.length - 7); } // eslint-disable-next-line camelcase if (execution_mode !== 'test' && !uri.startsWith('data:')) { return null; } const img = new Image(); await new Promise((r) => { img.onload = r; img.src = uri; }); return img; } async function predictUri(uri: string, uribg: string, bgoff: string | null) { const img = await imageFromUri(uri); const bg = uribg ? await imageFromUri(uribg) : null; const off = bgoff ? parseInt(bgoff) : null; return await predict(img!, bg!, off!); } const solveButton = document.createElement('input'); solveButton.id = 't-auto-solve'; solveButton.value = 'Solve'; solveButton.type = 'button'; solveButton.style.fontSize = '11px'; solveButton.style.padding = '0 2px'; solveButton.style.margin = '0px 0px 0px 6px'; solveButton.style.height = '18px'; solveButton.onclick = async function () { solve(true); }; const altsDiv = document.createElement('div'); altsDiv.id = 't-auto-options'; altsDiv.style.margin = '0'; altsDiv.style.padding = '0'; let storedPalceholder: string; let overrides = {}; function placeAfter(elem: HTMLElement, sibling: HTMLElement) { if (elem.parentElement !== sibling.parentElement) { setTimeout(function () { sibling.parentElement?.insertBefore(elem, sibling.nextElementSibling); }, 1); } } let previousText: string | null = null; async function solve(force?: any) { const resp = document.getElementById('t-resp') as HTMLInputElement; if (!resp) return; const bg = document.getElementById('t-bg'); if (!bg) return; const fg = document.getElementById('t-fg'); if (!fg) return; const help = document.getElementById('t-help'); if (!help) return; await backendloaded; placeAfter(solveButton, resp); placeAfter(altsDiv, help); // palememe setTimeout(function () { toggle(solveButton, bg.style.backgroundImage); }, 1); const text = fg.style.backgroundImage; if (!text) { altsDiv.innerHTML = ''; return; } if (text === previousText && !force) return; previousText = text; altsDiv.innerHTML = ''; if (!storedPalceholder) storedPalceholder = resp.placeholder; resp.placeholder = 'solving captcha...'; overrides = {}; const result = await predictUri( text, bg.style.backgroundImage, force ? bg.style.backgroundPositionX : null ); resp.placeholder = storedPalceholder; resp.value = result; } const observer = new MutationObserver(async function (mutationsList, observer) { solve(false); }); //window['solve'] = solve; observer.observe(document.body, { attributes: true, childList: true, subtree: true });