You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
477 lines
13 KiB
477 lines
13 KiB
import * as tf from '@tensorflow/tfjs';
|
|
import { setWasmPaths } from '@tensorflow/tfjs-backend-wasm';
|
|
import modelJSON from './model.json';
|
|
|
|
const charset = [' ', '0', '2', '4', '8', 'A', 'D', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'R', 'S', 'T', 'V', 'W', 'X', 'Y'];
|
|
let weightsData: Uint8Array; // base64 encoded weights
|
|
let model: tf.LayersModel;
|
|
|
|
tf.enableProdMode();
|
|
|
|
const wasmToUrl = (wasm: any) => {
|
|
const blb = new Blob([wasm], { type: 'application/wasm' });
|
|
return URL.createObjectURL(blb);
|
|
};
|
|
|
|
const backendloaded = (async () => {
|
|
try {
|
|
// dead code elimination should occur here
|
|
// eslint-disable-next-line camelcase
|
|
if (execution_mode === 'userscript' || execution_mode === 'test') {
|
|
weightsData = (await import('./group1-shard1of1.bin')).default;
|
|
const tfwasmthreadedsimd = (await import('./tfjs-backend-wasm-threaded-simd.wasm')).default;
|
|
const tfwasmsimd = (await import('./tfjs-backend-wasm-simd.wasm')).default;
|
|
const tfwasm = (await import('./tfjs-backend-wasm.wasm')).default;
|
|
setWasmPaths({
|
|
'tfjs-backend-wasm.wasm': wasmToUrl(tfwasm),
|
|
'tfjs-backend-wasm-simd.wasm': wasmToUrl(tfwasmsimd),
|
|
'tfjs-backend-wasm-threaded-simd.wasm': wasmToUrl(tfwasmthreadedsimd)
|
|
});
|
|
} else {
|
|
weightsData = new Uint8Array(await (await fetch(chrome.runtime.getURL('./group1-shard1of1.bin'))).arrayBuffer());
|
|
const args = {
|
|
'tfjs-backend-wasm.wasm': chrome.runtime.getURL('tfjs-backend-wasm.wasm'),
|
|
'tfjs-backend-wasm-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-simd.wasm'),
|
|
'tfjs-backend-wasm-threaded-simd.wasm': chrome.runtime.getURL('tfjs-backend-wasm-threaded-simd.wasm')
|
|
};
|
|
setWasmPaths(args);
|
|
}
|
|
const l = await tf.setBackend('wasm');
|
|
console.log('tf backend loaded', l);
|
|
} catch (err) {
|
|
console.log('tf err', err);
|
|
}
|
|
})();
|
|
|
|
function toggle(obj: HTMLElement, v?: any) {
|
|
if (v) obj.style.display = '';
|
|
else obj.style.display = 'none';
|
|
}
|
|
|
|
const iohander: tf.io.IOHandler = {
|
|
load: function () {
|
|
return new Promise((resolve, reject) => {
|
|
resolve({
|
|
modelTopology: modelJSON.modelTopology,
|
|
weightSpecs: modelJSON.weightsManifest[0].weights as tf.io.WeightsManifestEntry[],
|
|
weightData: weightsData.buffer,
|
|
format: modelJSON.format,
|
|
generatedBy: modelJSON.generatedBy,
|
|
convertedBy: modelJSON.convertedBy
|
|
});
|
|
});
|
|
}
|
|
};
|
|
|
|
async function load() {
|
|
model = await tf.loadLayersModel(iohander);
|
|
return model;
|
|
}
|
|
|
|
function black(x: number) {
|
|
return x < 64;
|
|
}
|
|
|
|
// Calculates "disorder" of the image. "Disorder" is the percentage of black pixels that have a
|
|
// non-black pixel below them. Minimizing this seems to be good enough metric for solving the slider.
|
|
function calculateDisorder(imgdata: ImageData) {
|
|
const a = imgdata.data;
|
|
const w = imgdata.width;
|
|
const h = imgdata.height;
|
|
const pic: number[] = [];
|
|
const visited: number[] = [];
|
|
|
|
for (let c = 0; c < w * h; c++) {
|
|
if (visited[c]) continue;
|
|
if (!black(a[c * 4])) continue;
|
|
|
|
let blackCount = 0;
|
|
const items: number[] = [];
|
|
const toVisit: number[] = [c];
|
|
while (toVisit.length > 0) {
|
|
const cc = toVisit[toVisit.length - 1];
|
|
toVisit.splice(toVisit.length - 1, 1);
|
|
|
|
if (visited[cc]) continue;
|
|
visited[cc] = 1;
|
|
|
|
if (black(a[cc * 4])) {
|
|
items.push(cc);
|
|
|
|
blackCount++;
|
|
toVisit.push(cc + 1);
|
|
toVisit.push(cc - 1);
|
|
toVisit.push(cc + w);
|
|
toVisit.push(cc - w);
|
|
}
|
|
}
|
|
|
|
if (blackCount >= 24) {
|
|
items.forEach(function (x) {
|
|
pic[x] = 1;
|
|
});
|
|
}
|
|
}
|
|
|
|
let res = 0;
|
|
let total = 0;
|
|
for (let c = 0; c < w * h - w; c++) {
|
|
if (pic[c] !== pic[c + w]) res += 1;
|
|
if (pic[c]) total += 1;
|
|
}
|
|
|
|
return res / (total === 0 ? 1 : total);
|
|
}
|
|
|
|
/*
|
|
* decide if a pixel is closer to black than to white.
|
|
* return 0 for white, 1 for black
|
|
*/
|
|
function pxlBlackOrWhite(r: number, g: number, b: number) {
|
|
return (r + g + b > 384) ? 0 : 1;
|
|
}
|
|
|
|
/*
|
|
* Get bordering pixels of transparent areas (the outline of the circles)
|
|
* and return their coordinates with the neighboring color.
|
|
*/
|
|
function getBoundries(imgdata: ImageData) {
|
|
const data = imgdata.data;
|
|
const width = imgdata.width;
|
|
|
|
let i = data.length - 1;
|
|
let cl = 0;
|
|
let cr = 0;
|
|
const chkArray = [];
|
|
let opq = true;
|
|
while (i > 0) {
|
|
// alpha channel above 128 is assumed opaque
|
|
const a = data[i] > 128;
|
|
if (a !== opq) {
|
|
if ((data[i - 4] > 128) === opq) {
|
|
// ignore just 1-width areas
|
|
i -= 4;
|
|
continue;
|
|
}
|
|
if (a) {
|
|
/* transparent pixel to its right */
|
|
/*
|
|
// set to color blue (for debugging)
|
|
data[i + 4] = 255;
|
|
data[i + 3] = 255;
|
|
data[i + 2] = 0;
|
|
data[i + 1] = 0;
|
|
*/
|
|
const pos = (i + 1) / 4;
|
|
const x = pos % width;
|
|
const y = (pos - x) / width;
|
|
// 1: black, 0: white
|
|
const clr = pxlBlackOrWhite(data[i - 1], data[i - 2], data[i - 3]);
|
|
chkArray.push([x, y, clr]);
|
|
cr += 1;
|
|
} else {
|
|
/* opaque pixel to its right */
|
|
/*
|
|
// set to color red (for debugging)
|
|
data[i] = 255;
|
|
data[i - 1] = 0;
|
|
data[i - 2] = 0;
|
|
data[i - 3] = 255;
|
|
*/
|
|
const pos = (i - 3) / 4;
|
|
const x = pos % width;
|
|
const y = (pos - x) / width;
|
|
// 1: black, 0: white
|
|
const clr = pxlBlackOrWhite(data[i + 1], data[i + 2], data[i + 3]);
|
|
chkArray.push([x, y, clr]);
|
|
cl += 1;
|
|
}
|
|
opq = a;
|
|
}
|
|
i -= 4;
|
|
}
|
|
return chkArray;
|
|
}
|
|
/*
|
|
* slide the background image and compare the colors of the border pixels in
|
|
* chkArray, the position with the most matches wins
|
|
* Return in slider-percentage.
|
|
*/
|
|
function getBestPos(bgdata: ImageData, chkArray: number[][], slideWidth: number) {
|
|
const data = bgdata.data;
|
|
const width = bgdata.width;
|
|
let bestSimilarity = 0;
|
|
let bestPos = 0;
|
|
|
|
for (let s = 0; s <= slideWidth; s += 1) {
|
|
let similarity = 0;
|
|
const amount = chkArray.length;
|
|
for (let p = 0; p < amount; p += 1) {
|
|
const chk = chkArray[p];
|
|
const x = chk[0] + s;
|
|
const y = chk[1];
|
|
const clr = chk[2];
|
|
const off = (y * width + x) * 4;
|
|
const bgclr = pxlBlackOrWhite(data[off], data[off + 1], data[off + 2]);
|
|
if (bgclr === clr) {
|
|
similarity += 1;
|
|
}
|
|
}
|
|
if (similarity > bestSimilarity) {
|
|
bestSimilarity = similarity;
|
|
bestPos = s;
|
|
}
|
|
}
|
|
return bestPos / slideWidth * 100;
|
|
}
|
|
|
|
async function getImageDataFromURI(uri: string) {
|
|
const image = await imageFromUri(uri);
|
|
if (!image)
|
|
throw new Error("No image");
|
|
const canvas = document.createElement('canvas');
|
|
canvas.width = image.width;
|
|
canvas.height = image.height;
|
|
const ctx = canvas.getContext('2d')!;
|
|
ctx.drawImage(image, 0, 0);
|
|
return ctx.getImageData(0, 0, canvas.width, canvas.height);
|
|
}
|
|
|
|
async function slideCaptcha(tfgElement: HTMLElement, tbgElement: HTMLElement, sliderElement: HTMLInputElement) {
|
|
// get data uris for captcha back- and foreground
|
|
const tbgUri = tbgElement.style.backgroundImage.slice(5, -2);
|
|
const tfgUri = tfgElement.style.backgroundImage.slice(5, -2);
|
|
|
|
// load foreground (image with holes)
|
|
const igd = await getImageDataFromURI(tfgUri);
|
|
// get array with pixels of foreground
|
|
// that we compare to background
|
|
const chkArray = getBoundries(igd);
|
|
// load background (image that gets slid)
|
|
const sigd = await getImageDataFromURI(tbgUri);
|
|
const slideWidth = sigd.width - igd.width;
|
|
// slide, compare and get best matching position
|
|
const sliderPos = getBestPos(sigd, chkArray, slideWidth);
|
|
// slide in the UI
|
|
sliderElement.value = '' + sliderPos;
|
|
(sliderElement as any).dispatchEvent(new Event('input'), { bubbles: true });
|
|
return 0 - (sliderPos / 2);
|
|
}
|
|
|
|
// returns ImageData from captcha's background image, foreground image, and offset (ranging from 0 to -50)
|
|
async function imageFromCanvas(img: HTMLImageElement, bg: HTMLImageElement, off: number | null) {
|
|
const h = img.height;
|
|
const w = img.width;
|
|
const th = 80;
|
|
const ph = 0;
|
|
const pw = 16;
|
|
const scale = th / h;
|
|
|
|
const canvas = document.createElement('canvas');
|
|
const fcanvas = document.createElement('canvas');
|
|
const cw = w * scale + pw * 2;
|
|
canvas.width = cw >= 300 ? 300 : cw;
|
|
canvas.height = th;
|
|
|
|
fcanvas.width = 300;
|
|
fcanvas.height = 80;
|
|
|
|
const ctx = canvas.getContext('2d')!;
|
|
const fctx = fcanvas.getContext('2d')!; // used to contain the captcha stretched to 300w
|
|
|
|
ctx.fillStyle = 'rgb(238,238,238)';
|
|
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
|
|
ctx.translate(canvas.width / 2, canvas.height / 2);
|
|
|
|
const draw = function (off: number) {
|
|
if (bg) {
|
|
const border = 4;
|
|
ctx.drawImage(
|
|
bg,
|
|
-off + border,
|
|
0,
|
|
w - border * 2,
|
|
h,
|
|
-w / 2 + border,
|
|
-h / 2,
|
|
w - border * 2,
|
|
h
|
|
);
|
|
}
|
|
ctx.drawImage(img, -w / 2, -h / 2, w, h);
|
|
fctx.drawImage(canvas, 0, 0, 300, 80);
|
|
};
|
|
|
|
if (bg && off == null) {
|
|
off = await slideCaptcha(document.getElementById('t-fg')!, document.getElementById('t-bg')!, document.getElementById('t-slider') as HTMLInputElement);
|
|
}
|
|
draw(off || 0);
|
|
return fctx.getImageData(0, 0, 300, 80);
|
|
}
|
|
|
|
function toMonochromeFloat(px: Uint8ClampedArray) {
|
|
const ret = Array<number>(px.length >> 2);
|
|
for (let i = 0; i < px.length; i += 4) {
|
|
ret[i >> 2] = px[i] / 255;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
const greedyCTCDecode = (yPred: tf.Tensor<tf.Rank>) => tf.tidy(() => yPred.argMax(-1).arraySync());
|
|
|
|
function processCTCDecodedSequence(decodedSequence: number[], blankLabel = 0) {
|
|
const result = [];
|
|
let prevLabel = blankLabel;
|
|
|
|
for (const label of decodedSequence) {
|
|
if (label !== blankLabel && label !== prevLabel) {
|
|
result.push(label);
|
|
}
|
|
prevLabel = label;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function indicesToSymbols(decodedIndices: number[]) {
|
|
return decodedIndices.map(index => charset[index - 1] || '');
|
|
}
|
|
|
|
async function predict(img: HTMLImageElement, bg: HTMLImageElement, off: number) {
|
|
if (!model) {
|
|
model = await load();
|
|
}
|
|
const image = await imageFromCanvas(img, bg, off);
|
|
if (!image)
|
|
throw new Error("Failed to gen image");
|
|
const mono = toMonochromeFloat(image.data);
|
|
|
|
const filtered2 = tf.tensor3d(mono, [image.height, image.width, 1]);
|
|
const prediction = model.predict(filtered2.transpose([1, 0, 2]).expandDims(0));
|
|
|
|
let d: tf.TypedArray;
|
|
|
|
if (Array.isArray(prediction))
|
|
throw new Error("Unexpected inference type");
|
|
|
|
const v = greedyCTCDecode(prediction) as number[][];
|
|
const s = processCTCDecodedSequence(v[0], charset.length + 1);
|
|
return indicesToSymbols(s).join('').trim();
|
|
}
|
|
|
|
async function imageFromUri(uri: string) {
|
|
if (uri.startsWith('url("')) {
|
|
uri = uri.substr(5, uri.length - 7);
|
|
}
|
|
// eslint-disable-next-line camelcase
|
|
if (execution_mode !== 'test' && !uri.startsWith('data:')) {
|
|
return null;
|
|
}
|
|
|
|
const img = new Image();
|
|
await new Promise((r) => {
|
|
img.onload = r;
|
|
img.src = uri;
|
|
});
|
|
return img;
|
|
}
|
|
|
|
async function predictUri(uri: string, uribg: string, bgoff: string | null) {
|
|
const img = await imageFromUri(uri);
|
|
const bg = uribg ? await imageFromUri(uribg) : null;
|
|
const off = bgoff ? parseInt(bgoff) : null;
|
|
|
|
return await predict(img!, bg!, off!);
|
|
}
|
|
|
|
const solveButton = document.createElement('input');
|
|
solveButton.id = 't-auto-solve';
|
|
solveButton.value = 'Solve';
|
|
solveButton.type = 'button';
|
|
solveButton.style.fontSize = '11px';
|
|
solveButton.style.padding = '0 2px';
|
|
solveButton.style.margin = '0px 0px 0px 6px';
|
|
solveButton.style.height = '18px';
|
|
solveButton.onclick = async function () {
|
|
solve(true);
|
|
};
|
|
|
|
const altsDiv = document.createElement('div');
|
|
altsDiv.id = 't-auto-options';
|
|
altsDiv.style.margin = '0';
|
|
altsDiv.style.padding = '0';
|
|
|
|
let storedPalceholder: string;
|
|
|
|
let overrides = {};
|
|
|
|
function placeAfter(elem: HTMLElement, sibling: HTMLElement) {
|
|
if (elem.parentElement !== sibling.parentElement) {
|
|
setTimeout(function () {
|
|
sibling.parentElement?.insertBefore(elem, sibling.nextElementSibling);
|
|
}, 1);
|
|
}
|
|
}
|
|
|
|
let previousText: string | null = null;
|
|
async function solve(force?: any) {
|
|
const resp = document.getElementById('t-resp') as HTMLInputElement;
|
|
if (!resp) return;
|
|
|
|
const bg = document.getElementById('t-bg');
|
|
if (!bg) return;
|
|
|
|
const fg = document.getElementById('t-fg');
|
|
if (!fg) return;
|
|
|
|
const help = document.getElementById('t-help');
|
|
if (!help) return;
|
|
|
|
await backendloaded;
|
|
|
|
placeAfter(solveButton, resp);
|
|
placeAfter(altsDiv, help);
|
|
|
|
// palememe
|
|
setTimeout(function () {
|
|
toggle(solveButton, bg.style.backgroundImage);
|
|
}, 1);
|
|
|
|
const text = fg.style.backgroundImage;
|
|
if (!text) {
|
|
altsDiv.innerHTML = '';
|
|
return;
|
|
}
|
|
|
|
if (text === previousText && !force) return;
|
|
previousText = text;
|
|
|
|
altsDiv.innerHTML = '';
|
|
if (!storedPalceholder)
|
|
storedPalceholder = resp.placeholder;
|
|
resp.placeholder = 'solving captcha...';
|
|
|
|
overrides = {};
|
|
|
|
const result = await predictUri(
|
|
text,
|
|
bg.style.backgroundImage,
|
|
force ? bg.style.backgroundPositionX : null
|
|
);
|
|
resp.placeholder = storedPalceholder;
|
|
|
|
resp.value = result;
|
|
}
|
|
|
|
const observer = new MutationObserver(async function (mutationsList, observer) {
|
|
solve(false);
|
|
});
|
|
|
|
//window['solve'] = solve;
|
|
|
|
observer.observe(document.body, {
|
|
attributes: true,
|
|
childList: true,
|
|
subtree: true
|
|
});
|
|
|