import * as tf from '@tensorflow/tfjs' import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm' import charsetJSON from './charset.json' import modelJSON from './model.json' let weightsData let model tf.enableProdMode() const wasmToUrl = wasm => { const blb = new Blob([tfwasm]) return URL.createObjectURL(blb) } const backendloaded = (async () => { try { // dead code elimination should occur here // eslint-disable-next-line camelcase if (execution_mode === 'userscript') { weightsData = import('./model.weights.bin') const tfwasmthreadedsimd = await import('./tfjs-backend-wasm-threaded-simd.wasm') const tfwasmsimd = await import('./tfjs-backend-wasm-simd.wasm') const tfwasm = await import('./tfjs-backend-wasm.wasm') setWasmPaths({ 'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm), 'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd), 'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd) }) } else { weightsData = await (await fetch(chrome.runtime.getURL('./model.weights.bin'))).text() const args = { 'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'), 'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'), 'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm') } setWasmPaths(args) } const l = await tf.setBackend('wasm') console.log('tf backend loaded', l) } catch (err) { console.log('tf err', err) } })() function toggle (obj, v) { if (v) obj.style.display = '' else obj.style.display = 'none' } function base64ToArray (base64) { const binaryString = window.atob(base64) const len = binaryString.length const bytes = new Uint8Array(len) for (let i = 0; i < len; i++) { bytes[i] = binaryString.charCodeAt(i) } return bytes.buffer } const iohander = { load: function () { return new Promise((resolve, reject) => { resolve({ modelTopology: modelJSON.modelTopology, weightSpecs: modelJSON.weightsManifest[0].weights, weightData: base64ToArray(weightsData), format: modelJSON.format, generatedBy: modelJSON.generatedBy, convertedBy: modelJSON.convertedBy }) }) } } async function load () { const uploadJSONInput = document.getElementById('upload-json') const uploadWeightsInput = document.getElementById('upload-weights-1') model = await tf.loadLayersModel(iohander) return model } function black (x) { return x < 64 } // Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a // non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider. function calculateDisorder (imgdata) { const a = imgdata.data const w = imgdata.width const h = imgdata.height const pic = [] const visited = [] for (let c = 0; c < w * h; c++) { if (visited[c]) continue if (!black(a[c * 4])) continue let blackCount = 0 const items = [] const toVisit = [c] while (toVisit.length > 0) { const cc = toVisit[toVisit.length - 1] toVisit.splice(toVisit.length - 1, 1) if (visited[cc]) continue visited[cc] = 1 if (black(a[cc * 4])) { items.push(cc) blackCount++ toVisit.push(cc + 1) toVisit.push(cc - 1) toVisit.push(cc + w) toVisit.push(cc - w) } } if (blackCount >= 24) { items.forEach(function (x) { pic[x] = 1 }) } } let res = 0 let total = 0 for (let c = 0; c < w * h - w; c++) { if (pic[c] !== pic[c + w]) res += 1 if (pic[c]) total += 1 } return res / (total === 0 ? 1 : total) } // returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50) function imageFromCanvas (img, bg, off) { const h = img.height const w = img.width const th = 80 const ph = 0 const pw = 16 const scale = th / h const canvas = document.createElement('canvas') canvas.height = w * scale + pw * 2 canvas.width = th const ctx = canvas.getContext('2d') ctx.fillStyle = 'rgb(238,238,238)' ctx.fillRect(0, 0, canvas.width, canvas.height) ctx.translate(canvas.width / 2, canvas.height / 2) ctx.scale(-scale, scale) ctx.rotate((90 * Math.PI) / 180) const draw = function (off) { if (bg) { const border = 4 ctx.drawImage( bg, -off + border, 0, w - border * 2, h, -w / 2 + border, -h / 2, w - border * 2, h ) } ctx.drawImage(img, -w / 2, -h / 2, w, h) } // if off is not specified and background image is present, try to figure out // the best offset automatically; select the offset that has smallest value of // calculateDisorder for the resulting image if (bg && off == null) { let bestDisorder = 999 let bestImagedata = null let bestOff = -1 for (let off = 0; off >= -50; off--) { draw(off) const imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height) const disorder = calculateDisorder(imgdata) if (disorder < bestDisorder) { bestDisorder = disorder bestImagedata = imgdata bestOff = off } } // not the best idea to do this here setTimeout(function () { const bg = document.getElementById('t-bg') const slider = document.getElementById('t-slider') if (!bg || !slider) return slider.value = -bestOff * 2 bg.style.backgroundPositionX = bestOff + 'px' }, 1) return bestImagedata } else { draw(off) return ctx.getImageData(0, 0, canvas.width, canvas.height) } } // for debugging purposes function imagedataToImage (imagedata) { const canvas = document.createElement('canvas') const ctx = canvas.getContext('2d') canvas.width = imagedata.width canvas.height = imagedata.height ctx.putImageData(imagedata, 0, 0) const image = new Image() image.src = canvas.toDataURL() return image } async function predict (img, bg, off) { if (!model) { model = await load() } const image = imageFromCanvas(img, bg, off) const tensor = tf.browser .fromPixels(image, 1) .mul(-1 / 238) .add(1) const prediction = await model.predict(tensor.expandDims(0)).data() return createSequence(prediction) } function createSequence (prediction) { const csl = charsetJSON.charset.length const sequence = [] for (let pos = 0; pos < prediction.length; pos += csl) { const preds = prediction.slice(pos, pos + csl) const max = Math.max(...preds) const seqElem = {} for (let i = 0; i < csl; i++) { const p = preds[i] / max const c = charsetJSON.charset[i + 1] if (p >= 0.05) { seqElem[c || ''] = p } } sequence.push(seqElem) } return sequence } function postprocess (sequence, overrides) { const csl = charsetJSON.charset.length let possibilities = [{ sequence: [] }] sequence.forEach(function (e, i) { let additions if (overrides && overrides[i] !== undefined) { additions = [{ sym: overrides[i], off: i, conf: 1 }] } else { additions = Object.keys(e).map(function (sym) { return { sym, off: i, conf: e[sym] } }) } if (additions.length === 1 && additions[0].sym === '') return const oldpos = possibilities possibilities = [] oldpos.forEach(function (possibility) { additions.forEach(function (a) { const seq = [...possibility.sequence] if (a.sym !== '') seq.push([a.sym, a.off, a.conf]) const obj = { sequence: seq } possibilities.push(obj) }) }) }) const res = {} possibilities.forEach(function (p) { let line = '' let lastSym let lastOff = -1 let count = 0 let prob = 0 p.sequence.forEach(function (e) { const sym = e[0] const off = e[1] const conf = e[2] if (sym === lastSym && lastOff + 2 >= off) { return } line += sym lastSym = sym lastOff = off prob += conf count++ }) if (count > 0) prob /= count if (prob > res[line] || !res[line]) { res[line] = prob } }) let keys = Object.keys(res).sort(function (a, b) { return res[a] < res[b] }) const keysFitting = keys.filter(function (x) { return x.length === 5 || x.length === 6 }) if (keysFitting.length > 0) keys = keysFitting return keys.map(function (x) { return { seq: x, prob: res[x] } }) } async function imageFromUri (uri) { if (uri.startsWith('url("')) { uri = uri.substr(5, uri.length - 7) } if (!uri.startsWith('data:')) { return null } const img = new Image() await new Promise((r) => (img.onload = r), (img.src = uri)) return img } async function predictUri (uri, uribg, bgoff) { const img = await imageFromUri(uri) const bg = uribg ? await imageFromUri(uribg) : null const off = bgoff ? parseInt(bgoff) : null return await predict(img, bg, off) } const solveButton = document.createElement('input') solveButton.id = 't-auto-solve' solveButton.value = 'Solve' solveButton.type = 'button' solveButton.style.fontSize = '11px' solveButton.style.padding = '0 2px' solveButton.style.margin = '0px 0px 0px 6px' solveButton.style.height = '18px' solveButton.onclick = async function () { solve(true) } const altsDiv = document.createElement('div') altsDiv.id = 't-auto-options' altsDiv.style.margin = '0' altsDiv.style.padding = '0' let storedPalceholder let overrides = {} function placeAfter (elem, sibling) { if (elem.parentElement !== sibling.parentElement) { setTimeout(function () { sibling.parentElement.insertBefore(elem, sibling.nextElementSibling) }, 1) } } let previousText = null async function solve (force) { const resp = document.getElementById('t-resp') if (!resp) return const bg = document.getElementById('t-bg') if (!bg) return const fg = document.getElementById('t-fg') if (!fg) return const help = document.getElementById('t-help') if (!help) return await backendloaded placeAfter(solveButton, resp) placeAfter(altsDiv, help) // palememe setTimeout(function () { toggle(solveButton, bg.style.backgroundImage) }, 1) const text = fg.style.backgroundImage if (!text) { altsDiv.innerHTML = '' return } if (text === previousText && !force) return previousText = text altsDiv.innerHTML = '' if (!storedPalceholder) storedPalceholder = resp.placeholder resp.placeholder = 'solving captcha...' overrides = {} const sequence = await predictUri( text, bg.style.backgroundImage, force ? bg.style.backgroundPositionX : null ) const opts = postprocess(sequence) resp.placeholder = storedPalceholder showOpts(opts) } function showOpts (opts) { const resp = document.getElementById('t-resp') if (!resp) return altsDiv.innerHTML = '' if (opts.length === 0) { resp.value = '' return } resp.value = opts[0].seq // for now don't display options since it seems more difficult to pick than type the whole thing // eslint-disable-next-line no-constant-condition, no-empty if (opts.length === 1 || true) { } } const observer = new MutationObserver(async function (mutationsList, observer) { solve(false) }) observer.observe(document.body, { attributes: true, childList: true, subtree: true })