JKCS/src/main.js

import * as tf from '@tensorflow/tfjs'
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm'
import charsetJSON from './charset.json'
import modelJSON from './model.json'

let weightsData
let model

tf.enableProdMode()

const wasmToUrl = wasm => {
  const blb = new Blob([tfwasm])
  return URL.createObjectURL(blb)
}

const backendloaded = (async () => {
  try {
    // dead code elimination should occur here
    // eslint-disable-next-line camelcase
    if (execution_mode === 'userscript') {
      weightsData = import('./model.weights.bin')

      const tfwasmthreadedsimd = await import('./tfjs-backend-wasm-threaded-simd.wasm')
      const tfwasmsimd = await import('./tfjs-backend-wasm-simd.wasm')
      const tfwasm = await import('./tfjs-backend-wasm.wasm')
      setWasmPaths({
        'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm),
        'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd),
        'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd)
      })
    } else {
      weightsData = await (await fetch(chrome.runtime.getURL('./model.weights.bin'))).text()
      const args = {
        'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'),
        'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'),
        'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm')
      }
      setWasmPaths(args)
    }
    const l = await tf.setBackend('wasm')
    console.log('tf backend loaded', l)
  } catch (err) {
    console.log('tf err', err)
  }
})()

function toggle (obj, v) {
  if (v) obj.style.display = ''
  else obj.style.display = 'none'
}

function base64ToArray (base64) {
  const binaryString = window.atob(base64)
  const len = binaryString.length
  const bytes = new Uint8Array(len)
  for (let i = 0; i < len; i++) {
    bytes[i] = binaryString.charCodeAt(i)
  }
  return bytes.buffer
}

const iohander = {
  load: function () {
    return new Promise((resolve, reject) => {
      resolve({
        modelTopology: modelJSON.modelTopology,
        weightSpecs: modelJSON.weightsManifest[0].weights,
        weightData: base64ToArray(weightsData),
        format: modelJSON.format,
        generatedBy: modelJSON.generatedBy,
        convertedBy: modelJSON.convertedBy
      })
    })
  }
}

async function load () {
  const uploadJSONInput = document.getElementById('upload-json')
  const uploadWeightsInput = document.getElementById('upload-weights-1')
  model = await tf.loadLayersModel(iohander)
  return model
}

function black (x) {
  return x < 64
}

// Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a
// non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider.
function calculateDisorder (imgdata) {
  const a = imgdata.data
  const w = imgdata.width
  const h = imgdata.height
  const pic = []
  const visited = []

  for (let c = 0; c < w * h; c++) {
    if (visited[c]) continue
    if (!black(a[c * 4])) continue

    let blackCount = 0
    const items = []
    const toVisit = [c]
    while (toVisit.length > 0) {
      const cc = toVisit[toVisit.length - 1]
      toVisit.splice(toVisit.length - 1, 1)

      if (visited[cc]) continue
      visited[cc] = 1

      if (black(a[cc * 4])) {
        items.push(cc)

        blackCount++
        toVisit.push(cc + 1)
        toVisit.push(cc - 1)
        toVisit.push(cc + w)
        toVisit.push(cc - w)
      }
    }

    if (blackCount >= 24) {
      items.forEach(function (x) {
        pic[x] = 1
      })
    }
  }

  let res = 0
  let total = 0
  for (let c = 0; c < w * h - w; c++) {
    if (pic[c] !== pic[c + w]) res += 1
    if (pic[c]) total += 1
  }

  return res / (total === 0 ? 1 : total)
}

// returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50)
function imageFromCanvas (img, bg, off) {
  const h = img.height
  const w = img.width
  const th = 80
  const ph = 0
  const pw = 16
  const scale = th / h

  const canvas = document.createElement('canvas')
  canvas.height = w * scale + pw * 2
  canvas.width = th

  const ctx = canvas.getContext('2d')

  ctx.fillStyle = 'rgb(238,238,238)'
  ctx.fillRect(0, 0, canvas.width, canvas.height)

  ctx.translate(canvas.width / 2, canvas.height / 2)
  ctx.scale(-scale, scale)
  ctx.rotate((90 * Math.PI) / 180)

  const draw = function (off) {
    if (bg) {
      const border = 4
      ctx.drawImage(
        bg,
        -off + border,
        0,
        w - border * 2,
        h,
        -w / 2 + border,
        -h / 2,
        w - border * 2,
        h
      )
    }
    ctx.drawImage(img, -w / 2, -h / 2, w, h)
  }

  // if off is not specified and background image is present, try to figure out
  // the best offset automatically; select the offset that has smallest value of
  // calculateDisorder for the resulting image
  if (bg && off == null) {
    let bestDisorder = 999
    let bestImagedata = null
    let bestOff = -1

    for (let off = 0; off >= -50; off--) {
      draw(off)

      const imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height)
      const disorder = calculateDisorder(imgdata)

      if (disorder < bestDisorder) {
        bestDisorder = disorder
        bestImagedata = imgdata
        bestOff = off
      }
    }

    // not the best idea to do this here
    setTimeout(function () {
      const bg = document.getElementById('t-bg')
      const slider = document.getElementById('t-slider')
      if (!bg || !slider) return

      slider.value = -bestOff * 2
      bg.style.backgroundPositionX = bestOff + 'px'
    }, 1)

    return bestImagedata
  } else {
    draw(off)

    return ctx.getImageData(0, 0, canvas.width, canvas.height)
  }
}

// for debugging purposes
function imagedataToImage (imagedata) {
  const canvas = document.createElement('canvas')
  const ctx = canvas.getContext('2d')
  canvas.width = imagedata.width
  canvas.height = imagedata.height
  ctx.putImageData(imagedata, 0, 0)

  const image = new Image()
  image.src = canvas.toDataURL()
  return image
}

async function predict (img, bg, off) {
  if (!model) {
    model = await load()
  }
  const image = imageFromCanvas(img, bg, off)
  const tensor = tf.browser
    .fromPixels(image, 1)
    .mul(-1 / 238)
    .add(1)
  const prediction = await model.predict(tensor.expandDims(0)).data()

  return createSequence(prediction)
}

function createSequence (prediction) {
  const csl = charsetJSON.charset.length
  const sequence = []

  for (let pos = 0; pos < prediction.length; pos += csl) {
    const preds = prediction.slice(pos, pos + csl)
    const max = Math.max(...preds)

    const seqElem = {}

    for (let i = 0; i < csl; i++) {
      const p = preds[i] / max
      const c = charsetJSON.charset[i + 1]

      if (p >= 0.05) {
        seqElem[c || ''] = p
      }
    }

    sequence.push(seqElem)
  }

  return sequence
}

function postprocess (sequence, overrides) {
  const csl = charsetJSON.charset.length
  let possibilities = [{ sequence: [] }]

  sequence.forEach(function (e, i) {
    let additions
    if (overrides && overrides[i] !== undefined) {
      additions = [{ sym: overrides[i], off: i, conf: 1 }]
    } else {
      additions = Object.keys(e).map(function (sym) {
        return { sym, off: i, conf: e[sym] }
      })
    }

    if (additions.length === 1 && additions[0].sym === '') return

    const oldpos = possibilities
    possibilities = []
    oldpos.forEach(function (possibility) {
      additions.forEach(function (a) {
        const seq = [...possibility.sequence]
        if (a.sym !== '') seq.push([a.sym, a.off, a.conf])

        const obj = {
          sequence: seq
        }

        possibilities.push(obj)
      })
    })
  })

  const res = {}
  possibilities.forEach(function (p) {
    let line = ''
    let lastSym
    let lastOff = -1
    let count = 0
    let prob = 0

    p.sequence.forEach(function (e) {
      const sym = e[0]
      const off = e[1]
      const conf = e[2]

      if (sym === lastSym && lastOff + 2 >= off) {
        return
      }

      line += sym

      lastSym = sym
      lastOff = off
      prob += conf
      count++
    })

    if (count > 0) prob /= count

    if (prob > res[line] || !res[line]) {
      res[line] = prob
    }
  })

  let keys = Object.keys(res).sort(function (a, b) {
    return res[a] < res[b]
  })
  const keysFitting = keys.filter(function (x) {
    return x.length === 5 || x.length === 6
  })
  if (keysFitting.length > 0) keys = keysFitting

  return keys.map(function (x) {
    return { seq: x, prob: res[x] }
  })
}

async function imageFromUri (uri) {
  if (uri.startsWith('url("')) {
    uri = uri.substr(5, uri.length - 7)
  }
  if (!uri.startsWith('data:')) {
    return null
  }

  const img = new Image()
  await new Promise((r) => (img.onload = r), (img.src = uri))

  return img
}

async function predictUri (uri, uribg, bgoff) {
  const img = await imageFromUri(uri)
  const bg = uribg ? await imageFromUri(uribg) : null
  const off = bgoff ? parseInt(bgoff) : null

  return await predict(img, bg, off)
}

const solveButton = document.createElement('input')
solveButton.id = 't-auto-solve'
solveButton.value = 'Solve'
solveButton.type = 'button'
solveButton.style.fontSize = '11px'
solveButton.style.padding = '0 2px'
solveButton.style.margin = '0px 0px 0px 6px'
solveButton.style.height = '18px'
solveButton.onclick = async function () {
  solve(true)
}

const altsDiv = document.createElement('div')
altsDiv.id = 't-auto-options'
altsDiv.style.margin = '0'
altsDiv.style.padding = '0'

let storedPalceholder

let overrides = {}

function placeAfter (elem, sibling) {
  if (elem.parentElement !== sibling.parentElement) {
    setTimeout(function () {
      sibling.parentElement.insertBefore(elem, sibling.nextElementSibling)
    }, 1)
  }
}

let previousText = null
async function solve (force) {
  const resp = document.getElementById('t-resp')
  if (!resp) return

  const bg = document.getElementById('t-bg')
  if (!bg) return

  const fg = document.getElementById('t-fg')
  if (!fg) return

  const help = document.getElementById('t-help')
  if (!help) return

  await backendloaded

  placeAfter(solveButton, resp)
  placeAfter(altsDiv, help)

  // palememe
  setTimeout(function () {
    toggle(solveButton, bg.style.backgroundImage)
  }, 1)

  const text = fg.style.backgroundImage
  if (!text) {
    altsDiv.innerHTML = ''
    return
  }

  if (text === previousText && !force) return
  previousText = text

  altsDiv.innerHTML = ''
  if (!storedPalceholder) storedPalceholder = resp.placeholder
  resp.placeholder = 'solving captcha...'

  overrides = {}

  const sequence = await predictUri(
    text,
    bg.style.backgroundImage,
    force ? bg.style.backgroundPositionX : null
  )
  const opts = postprocess(sequence)
  resp.placeholder = storedPalceholder

  showOpts(opts)
}

function showOpts (opts) {
  const resp = document.getElementById('t-resp')
  if (!resp) return

  altsDiv.innerHTML = ''

  if (opts.length === 0) {
    resp.value = ''
    return
  }

  resp.value = opts[0].seq

  // for now don't display options since it seems more difficult to pick than type the whole thing
  // eslint-disable-next-line no-constant-condition, no-empty
  if (opts.length === 1 || true) {

  }
}

const observer = new MutationObserver(async function (mutationsList, observer) {
  solve(false)
})

observer.observe(document.body, {
  attributes: true,
  childList: true,
  subtree: true
})