import * as tf from '@tensorflow/tfjs'; import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'; import modelJSON from './model.json'; import ccl from './ccl'; const charset = [' ', '0', '2', '4', '5', '8', 'A', 'D', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'R', 'S', 'T', 'V', 'W', 'X', 'Y']; let weightsData; let model; tf.enableProdMode(); const wasmToUrl = wasm => { const blb = new Blob([wasm], { type: 'application/wasm' }); return URL.createObjectURL(blb); }; const backendloaded = (async () => { try { // dead code elimination should occur here // eslint-disable-next-line camelcase if (execution_mode === 'userscript' || execution_mode === 'test') { weightsData = (await import('./model.weights.bin')).default; const tfwasmthreadedsimd = (await import('./tfjs-backend-wasm-threaded-simd.wasm')).default; const tfwasmsimd = (await import('./tfjs-backend-wasm-simd.wasm')).default; const tfwasm = (await import('./tfjs-backend-wasm.wasm')).default; setWasmPaths({ 'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm), 'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd), 'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd) }); } else { weightsData = await (await fetch(chrome.runtime.getURL('./model.weights.bin'))).text(); const args = { 'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'), 'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'), 'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm') }; setWasmPaths(args); } const l = await tf.setBackend('wasm'); console.log('tf backend loaded', l); } catch (err) { console.log('tf err', err); } })(); function toggle(obj, v) { if (v) obj.style.display = ''; else obj.style.display = 'none'; } function base64ToArray(base64) { const binaryString = window.atob(base64); const len = binaryString.length; const bytes = new Uint8Array(len); for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i); } return bytes.buffer; } const iohander = { load: function () { return new Promise((resolve, reject) => { resolve({ modelTopology: modelJSON.modelTopology, weightSpecs: modelJSON.weightsManifest[0].weights, weightData: base64ToArray(weightsData), format: modelJSON.format, generatedBy: modelJSON.generatedBy, convertedBy: modelJSON.convertedBy }); }); } }; async function load() { const uploadJSONInput = document.getElementById('upload-json'); const uploadWeightsInput = document.getElementById('upload-weights-1'); model = await tf.loadLayersModel(iohander); return model; } function black(x) { return x < 64; } // Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a // non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider. function calculateDisorder(imgdata) { const a = imgdata.data; const w = imgdata.width; const h = imgdata.height; const pic = []; const visited = []; for (let c = 0; c < w * h; c++) { if (visited[c]) continue; if (!black(a[c * 4])) continue; let blackCount = 0; const items = []; const toVisit = [c]; while (toVisit.length > 0) { const cc = toVisit[toVisit.length - 1]; toVisit.splice(toVisit.length - 1, 1); if (visited[cc]) continue; visited[cc] = 1; if (black(a[cc * 4])) { items.push(cc); blackCount++; toVisit.push(cc + 1); toVisit.push(cc - 1); toVisit.push(cc + w); toVisit.push(cc - w); } } if (blackCount >= 24) { items.forEach(function (x) { pic[x] = 1; }); } } let res = 0; let total = 0; for (let c = 0; c < w * h - w; c++) { if (pic[c] !== pic[c + w]) res += 1; if (pic[c]) total += 1; } return res / (total === 0 ? 1 : total); } // returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50) function imageFromCanvas(img, bg, off) { const h = img.height; const w = img.width; const th = 80; const ph = 0; const pw = 16; const scale = th / h; const canvas = document.createElement('canvas'); canvas.height = w * scale + pw * 2; canvas.width = th; const ctx = canvas.getContext('2d', { willReadFrequently: true }); ctx.fillStyle = 'rgb(238,238,238)'; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.translate(canvas.width / 2, canvas.height / 2); ctx.scale(-scale, scale); ctx.rotate((90 * Math.PI) / 180); const adf = 1 / 3; const draw = function (off) { if (bg) { const border = 4; ctx.drawImage( bg, -off + border, 0, w - border * 2, h, -w / 2 + border, -h / 2, w - border * 2, h ); } ctx.drawImage(img, -w / 2, -h / 2, w, h); }; // if off is not specified and background image is present, try to figure out // the best offset automatically; select the offset that has smallest value of // calculateDisorder for the resulting image if (bg && off == null) { let bestDisorder = 999; let bestImagedata = null; let bestOff = -1; for (let off = 0; off >= -50; off--) { draw(off); let imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height); const disorder = calculateDisorder(imgdata); if (disorder < bestDisorder) { bestDisorder = disorder; draw(off); imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height); bestImagedata = imgdata; bestOff = off; } } // not the best idea to do this here setTimeout(function () { const bg = document.getElementById('t-bg'); const slider = document.getElementById('t-slider'); if (!bg || !slider) return; slider.value = -bestOff * 2; bg.style.backgroundPositionX = bestOff + 'px'; }, 1); draw(bestOff); return bestImagedata; } else { draw(off); return ctx.getImageData(0, 0, canvas.width, canvas.height); } } // for debugging purposes function imagedataToImage(imagedata) { const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); canvas.width = imagedata.width; canvas.height = imagedata.height; ctx.putImageData(imagedata, 0, 0); const image = new Image(); image.src = canvas.toDataURL(); return image; } async function predict(img, bg, off) { if (!model) { model = await load(); } const image = imageFromCanvas(img, bg, off); const labels = ccl.connectedComponentLabeling(image.data.map(e => +(e > 128)), image.width, image.height); const props = ccl.computeBounds(labels, image.width, image.height); const sortedByArea = Object.entries(props).sort((a, b) => a[1].area - b[1].area); const eightBiggest = sortedByArea.slice(-8); const filtered = new Float32Array(80 * 300); // TODO: maybe centering? for (const [label, region] of eightBiggest) { if ((region.maxRow - region.minRow) <= 20) { continue; } for (let y = region.minRow; y < region.maxRow; ++y) { for (let x = region.minCol; y < region.maxCol; ++x) { if (labels[y * image.width + x] === label) { filtered[y * 300 + x] = 1; } } } } const tensor = tf.tensor3d(filtered, [80, 300, 1], 'float32'); const prediction = await model.predict(tensor.expandDims(0)).data(); return createSequence(prediction); } function createSequence(prediction) { const csl = charset.length; const sequence = []; // for each prediction for (let pos = 0; pos < prediction.length; pos += csl) { // look at the probabilities for the 22 token characters const preds = prediction.slice(pos, pos + csl); const max = Math.max(...preds); const seqElem = {}; for (let i = 0; i < csl; i++) { const p = preds[i] / max; // normalize probability const c = charset[i + 1]; if (p >= 0.05) { // if it's probable enough seqElem[c || ''] = p; // save its probability, to give alternative solutions } } sequence.push(seqElem); } return sequence; } function postprocess(sequence, overrides) { const csl = charset.length; let possibilities = [{ sequence: [] }]; sequence.forEach(function (e, i) { let additions; if (overrides && overrides[i] !== undefined) { additions = [{ sym: overrides[i], off: i, conf: 1 }]; } else { additions = Object.keys(e).map(function (sym) { return { sym, off: i, conf: e[sym] }; }); } if (additions.length === 1 && additions[0].sym === '') return; const oldpos = possibilities; possibilities = []; oldpos.forEach(function (possibility) { additions.forEach(function (a) { const seq = [...possibility.sequence]; if (a.sym !== '') seq.push([a.sym, a.off, a.conf]); const obj = { sequence: seq }; possibilities.push(obj); }); }); }); const res = {}; possibilities.forEach(function (p) { let line = ''; let lastSym; let lastOff = -1; let count = 0; let prob = 0; p.sequence.forEach(function (e) { const sym = e[0]; const off = e[1]; const conf = e[2]; if (sym === lastSym && lastOff + 2 >= off) { return; } line += sym; lastSym = sym; lastOff = off; prob += conf; count++; }); if (count > 0) prob /= count; if (prob > res[line] || !res[line]) { res[line] = prob; } }); let keys = Object.keys(res).sort(function (a, b) { return res[a] < res[b]; }); const keysFitting = keys.filter(function (x) { return x.length === 5 || x.length === 6; }); if (keysFitting.length > 0) keys = keysFitting; return keys.map(function (x) { return { seq: x, prob: res[x] }; }); } async function imageFromUri(uri) { if (uri.startsWith('url("')) { uri = uri.substr(5, uri.length - 7); } // eslint-disable-next-line camelcase if (execution_mode !== 'test' && !uri.startsWith('data:')) { return null; } const img = new Image(); await new Promise((r) => (img.onload = r), (img.src = uri)); return img; } async function predictUri(uri, uribg, bgoff) { const img = await imageFromUri(uri); const bg = uribg ? await imageFromUri(uribg) : null; const off = bgoff ? parseInt(bgoff) : null; return await predict(img, bg, off); } const solveButton = document.createElement('input'); solveButton.id = 't-auto-solve'; solveButton.value = 'Solve'; solveButton.type = 'button'; solveButton.style.fontSize = '11px'; solveButton.style.padding = '0 2px'; solveButton.style.margin = '0px 0px 0px 6px'; solveButton.style.height = '18px'; solveButton.onclick = async function () { solve(true); }; const altsDiv = document.createElement('div'); altsDiv.id = 't-auto-options'; altsDiv.style.margin = '0'; altsDiv.style.padding = '0'; let storedPalceholder; let overrides = {}; function placeAfter(elem, sibling) { if (elem.parentElement !== sibling.parentElement) { setTimeout(function () { sibling.parentElement.insertBefore(elem, sibling.nextElementSibling); }, 1); } } let previousText = null; async function solve(force) { const resp = document.getElementById('t-resp'); if (!resp) return; const bg = document.getElementById('t-bg'); if (!bg) return; const fg = document.getElementById('t-fg'); if (!fg) return; const help = document.getElementById('t-help'); if (!help) return; await backendloaded; placeAfter(solveButton, resp); placeAfter(altsDiv, help); // palememe setTimeout(function () { toggle(solveButton, bg.style.backgroundImage); }, 1); const text = fg.style.backgroundImage; if (!text) { altsDiv.innerHTML = ''; return; } if (text === previousText && !force) return; previousText = text; altsDiv.innerHTML = ''; if (!storedPalceholder) storedPalceholder = resp.placeholder; resp.placeholder = 'solving captcha...'; overrides = {}; const sequence = await predictUri( text, bg.style.backgroundImage, force ? bg.style.backgroundPositionX : null ); const opts = postprocess(sequence); resp.placeholder = storedPalceholder; showOpts(opts); } function showOpts(opts) { const resp = document.getElementById('t-resp'); if (!resp) return; altsDiv.innerHTML = ''; if (opts.length === 0) { resp.value = ''; return; } resp.value = opts[0].seq; // for now don't display options since it seems more difficult to pick than type the whole thing // eslint-disable-next-line no-constant-condition, no-empty if (opts.length === 1 || true) { } } const observer = new MutationObserver(async function (mutationsList, observer) { solve(false); }); window.solve = solve; observer.observe(document.body, { attributes: true, childList: true, subtree: true });