JKCS/src/main.js

import * as tf from '@tensorflow/tfjs';
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
import modelJSON from './model.json';
import ccl from './ccl';

const charset = [' ', '0', '2', '4', '5', '8', 'A', 'D', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'R', 'S', 'T', 'V', 'W', 'X', 'Y'];
let weightsData;
let model;

tf.enableProdMode();

const wasmToUrl = wasm => {
  const blb = new Blob([wasm], { type: 'application/wasm' });
  return URL.createObjectURL(blb);
};

const backendloaded = (async () => {
  try {
    // dead code elimination should occur here
    // eslint-disable-next-line camelcase
    if (execution_mode === 'userscript' || execution_mode === 'test') {
      weightsData = (await import('./model.weights.bin')).default;
      const tfwasmthreadedsimd = (await import('./tfjs-backend-wasm-threaded-simd.wasm')).default;
      const tfwasmsimd = (await import('./tfjs-backend-wasm-simd.wasm')).default;
      const tfwasm = (await import('./tfjs-backend-wasm.wasm')).default;
      setWasmPaths({
        'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm),
        'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd),
        'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd)
      });
    } else {
      weightsData = await (await fetch(chrome.runtime.getURL('./model.weights.bin'))).text();
      const args = {
        'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'),
        'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'),
        'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm')
      };
      setWasmPaths(args);
    }
    const l = await tf.setBackend('wasm');
    console.log('tf backend loaded', l);
  } catch (err) {
    console.log('tf err', err);
  }
})();

function toggle(obj, v) {
  if (v) obj.style.display = '';
  else obj.style.display = 'none';
}

function base64ToArray(base64) {
  const binaryString = window.atob(base64);
  const len = binaryString.length;
  const bytes = new Uint8Array(len);
  for (let i = 0; i < len; i++) {
    bytes[i] = binaryString.charCodeAt(i);
  }
  return bytes.buffer;
}

const iohander = {
  load: function () {
    return new Promise((resolve, reject) => {
      resolve({
        modelTopology: modelJSON.modelTopology,
        weightSpecs: modelJSON.weightsManifest[0].weights,
        weightData: base64ToArray(weightsData),
        format: modelJSON.format,
        generatedBy: modelJSON.generatedBy,
        convertedBy: modelJSON.convertedBy
      });
    });
  }
};

async function load() {
  const uploadJSONInput = document.getElementById('upload-json');
  const uploadWeightsInput = document.getElementById('upload-weights-1');
  model = await tf.loadLayersModel(iohander);
  return model;
}

function black(x) {
  return x < 64;
}

// Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a
// non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider.
function calculateDisorder(imgdata) {
  const a = imgdata.data;
  const w = imgdata.width;
  const h = imgdata.height;
  const pic = [];
  const visited = [];

  for (let c = 0; c < w * h; c++) {
    if (visited[c]) continue;
    if (!black(a[c * 4])) continue;

    let blackCount = 0;
    const items = [];
    const toVisit = [c];
    while (toVisit.length > 0) {
      const cc = toVisit[toVisit.length - 1];
      toVisit.splice(toVisit.length - 1, 1);

      if (visited[cc]) continue;
      visited[cc] = 1;

      if (black(a[cc * 4])) {
        items.push(cc);

        blackCount++;
        toVisit.push(cc + 1);
        toVisit.push(cc - 1);
        toVisit.push(cc + w);
        toVisit.push(cc - w);
      }
    }

    if (blackCount >= 24) {
      items.forEach(function (x) {
        pic[x] = 1;
      });
    }
  }

  let res = 0;
  let total = 0;
  for (let c = 0; c < w * h - w; c++) {
    if (pic[c] !== pic[c + w]) res += 1;
    if (pic[c]) total += 1;
  }

  return res / (total === 0 ? 1 : total);
}

// returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50)
function imageFromCanvas(img, bg, off) {
  const h = img.height;
  const w = img.width;
  const th = 80;
  const ph = 0;
  const pw = 16;
  const scale = th / h;

  const canvas = document.createElement('canvas');
  canvas.height = w * scale + pw * 2;
  canvas.width = th;

  const ctx = canvas.getContext('2d', { willReadFrequently: true });

  ctx.fillStyle = 'rgb(238,238,238)';
  ctx.fillRect(0, 0, canvas.width, canvas.height);

  ctx.translate(canvas.width / 2, canvas.height / 2);
  ctx.scale(-scale, scale);
  ctx.rotate((90 * Math.PI) / 180);

  const adf = 1 / 3;

  const draw = function (off) {
    if (bg) {
      const border = 4;
      ctx.drawImage(
        bg,
        -off + border,
        0,
        w - border * 2,
        h,
        -w / 2 + border,
        -h / 2,
        w - border * 2,
        h
      );
    }
    ctx.drawImage(img, -w / 2, -h / 2, w, h);
  };

  // if off is not specified and background image is present, try to figure out
  // the best offset automatically; select the offset that has smallest value of
  // calculateDisorder for the resulting image
  if (bg && off == null) {
    let bestDisorder = 999;
    let bestImagedata = null;
    let bestOff = -1;

    for (let off = 0; off >= -50; off--) {
      draw(off);

      let imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height);
      const disorder = calculateDisorder(imgdata);

      if (disorder < bestDisorder) {
        bestDisorder = disorder;
        draw(off);
        imgdata = ctx.getImageData(0, 0, canvas.width, canvas.height);
        bestImagedata = imgdata;
        bestOff = off;
      }
    }

    // not the best idea to do this here
    setTimeout(function () {
      const bg = document.getElementById('t-bg');
      const slider = document.getElementById('t-slider');
      if (!bg || !slider) return;

      slider.value = -bestOff * 2;
      bg.style.backgroundPositionX = bestOff + 'px';
    }, 1);
    draw(bestOff);
    return bestImagedata;
  } else {
    draw(off);
    return ctx.getImageData(0, 0, canvas.width, canvas.height);
  }
}

// for debugging purposes
function imagedataToImage(imagedata) {
  const canvas = document.createElement('canvas');
  const ctx = canvas.getContext('2d');
  canvas.width = imagedata.width;
  canvas.height = imagedata.height;
  ctx.putImageData(imagedata, 0, 0);

  const image = new Image();
  image.src = canvas.toDataURL();
  return image;
}

async function predict(img, bg, off) {
  if (!model) {
    model = await load();
  }
  const image = imageFromCanvas(img, bg, off);
  const labels = ccl.connectedComponentLabeling(image.data.map(e => +(e > 128)), image.width, image.height);
  const props = ccl.computeBounds(labels, image.width, image.height);

  const sortedByArea = Object.entries(props).sort((a, b) => a[1].area - b[1].area);
  const eightBiggest = sortedByArea.slice(-8);
  const filtered = new Float32Array(80 * 300);

  // TODO: maybe centering?
  for (const [label, region] of eightBiggest) {
    if ((region.maxRow - region.minRow) <= 20) {
      continue;
    }

    for (let y = region.minRow; y < region.maxRow; ++y) {
      for (let x = region.minCol; y < region.maxCol; ++x) {
        if (labels[y * image.width + x] === label) {
          filtered[y * 300 + x] = 1;
        }
      }
    }
  }

  const tensor = tf.tensor3d(filtered, [80, 300, 1], 'float32');
  const prediction = await model.predict(tensor.expandDims(0)).data();

  return createSequence(prediction);
}

function createSequence(prediction) {
  const csl = charset.length;
  const sequence = [];

  // for each prediction
  for (let pos = 0; pos < prediction.length; pos += csl) {
    // look at the probabilities for the 22 token characters
    const preds = prediction.slice(pos, pos + csl);
    const max = Math.max(...preds);

    const seqElem = {};

    for (let i = 0; i < csl; i++) {
      const p = preds[i] / max; // normalize probability
      const c = charset[i + 1];

      if (p >= 0.05) { // if it's probable enough
        seqElem[c || ''] = p; // save its probability, to give alternative solutions
      }
    }

    sequence.push(seqElem);
  }

  return sequence;
}

function postprocess(sequence, overrides) {
  const csl = charset.length;
  let possibilities = [{ sequence: [] }];

  sequence.forEach(function (e, i) {
    let additions;
    if (overrides && overrides[i] !== undefined) {
      additions = [{ sym: overrides[i], off: i, conf: 1 }];
    } else {
      additions = Object.keys(e).map(function (sym) {
        return { sym, off: i, conf: e[sym] };
      });
    }

    if (additions.length === 1 && additions[0].sym === '') return;

    const oldpos = possibilities;
    possibilities = [];
    oldpos.forEach(function (possibility) {
      additions.forEach(function (a) {
        const seq = [...possibility.sequence];
        if (a.sym !== '') seq.push([a.sym, a.off, a.conf]);

        const obj = {
          sequence: seq
        };

        possibilities.push(obj);
      });
    });
  });

  const res = {};
  possibilities.forEach(function (p) {
    let line = '';
    let lastSym;
    let lastOff = -1;
    let count = 0;
    let prob = 0;

    p.sequence.forEach(function (e) {
      const sym = e[0];
      const off = e[1];
      const conf = e[2];

      if (sym === lastSym && lastOff + 2 >= off) {
        return;
      }

      line += sym;

      lastSym = sym;
      lastOff = off;
      prob += conf;
      count++;
    });

    if (count > 0) prob /= count;

    if (prob > res[line] || !res[line]) {
      res[line] = prob;
    }
  });

  let keys = Object.keys(res).sort(function (a, b) {
    return res[a] < res[b];
  });
  const keysFitting = keys.filter(function (x) {
    return x.length === 5 || x.length === 6;
  });
  if (keysFitting.length > 0) keys = keysFitting;

  return keys.map(function (x) {
    return { seq: x, prob: res[x] };
  });
}

async function imageFromUri(uri) {
  if (uri.startsWith('url("')) {
    uri = uri.substr(5, uri.length - 7);
  }
  // eslint-disable-next-line camelcase
  if (execution_mode !== 'test' && !uri.startsWith('data:')) {
    return null;
  }

  const img = new Image();
  await new Promise((r) => (img.onload = r), (img.src = uri));
  return img;
}

async function predictUri(uri, uribg, bgoff) {
  const img = await imageFromUri(uri);
  const bg = uribg ? await imageFromUri(uribg) : null;
  const off = bgoff ? parseInt(bgoff) : null;

  return await predict(img, bg, off);
}

const solveButton = document.createElement('input');
solveButton.id = 't-auto-solve';
solveButton.value = 'Solve';
solveButton.type = 'button';
solveButton.style.fontSize = '11px';
solveButton.style.padding = '0 2px';
solveButton.style.margin = '0px 0px 0px 6px';
solveButton.style.height = '18px';
solveButton.onclick = async function () {
  solve(true);
};

const altsDiv = document.createElement('div');
altsDiv.id = 't-auto-options';
altsDiv.style.margin = '0';
altsDiv.style.padding = '0';

let storedPalceholder;

let overrides = {};

function placeAfter(elem, sibling) {
  if (elem.parentElement !== sibling.parentElement) {
    setTimeout(function () {
      sibling.parentElement.insertBefore(elem, sibling.nextElementSibling);
    }, 1);
  }
}

let previousText = null;
async function solve(force) {
  const resp = document.getElementById('t-resp');
  if (!resp) return;

  const bg = document.getElementById('t-bg');
  if (!bg) return;

  const fg = document.getElementById('t-fg');
  if (!fg) return;

  const help = document.getElementById('t-help');
  if (!help) return;

  await backendloaded;

  placeAfter(solveButton, resp);
  placeAfter(altsDiv, help);

  // palememe
  setTimeout(function () {
    toggle(solveButton, bg.style.backgroundImage);
  }, 1);

  const text = fg.style.backgroundImage;
  if (!text) {
    altsDiv.innerHTML = '';
    return;
  }

  if (text === previousText && !force) return;
  previousText = text;

  altsDiv.innerHTML = '';
  if (!storedPalceholder) storedPalceholder = resp.placeholder;
  resp.placeholder = 'solving captcha...';

  overrides = {};

  const sequence = await predictUri(
    text,
    bg.style.backgroundImage,
    force ? bg.style.backgroundPositionX : null
  );
  const opts = postprocess(sequence);
  resp.placeholder = storedPalceholder;

  showOpts(opts);
}

function showOpts(opts) {
  const resp = document.getElementById('t-resp');
  if (!resp) return;

  altsDiv.innerHTML = '';

  if (opts.length === 0) {
    resp.value = '';
    return;
  }

  resp.value = opts[0].seq;

  // for now don't display options since it seems more difficult to pick than type the whole thing
  // eslint-disable-next-line no-constant-condition, no-empty
  if (opts.length === 1 || true) {

  }
}

const observer = new MutationObserver(async function (mutationsList, observer) {
  solve(false);
});
window.solve = solve;
observer.observe(document.body, {
  attributes: true,
  childList: true,
  subtree: true
});