Spaces:
Running
Running
Update script.js
Browse files
script.js
CHANGED
|
@@ -11,7 +11,7 @@ tokenizeButton.addEventListener("click", async () => {
|
|
| 11 |
let tokenizer = new Tokenizer(tokenizerJSON, tokenizerConfig);
|
| 12 |
let text = document.querySelector("#text").value.replace(/[^\S ]/g, "");
|
| 13 |
let tokenized = tokenizer.encode(text);
|
| 14 |
-
document.querySelector("#totalLength").textContent = `Total length: ${tokenized.length}`;
|
| 15 |
let output = document.querySelector("#output");
|
| 16 |
output.contentDocument.body.innerHTML = "";
|
| 17 |
let colors = ["#ff0000", "#ff8700", "#ffd300", "#deff0a", "#a1ff0a", "#0aff99", "#0aefff", "#147df5", "#580aff", "#be0aff"].sort(() => Math.random() - 0.5);
|
|
|
|
| 11 |
let tokenizer = new Tokenizer(tokenizerJSON, tokenizerConfig);
|
| 12 |
let text = document.querySelector("#text").value.replace(/[^\S ]/g, "");
|
| 13 |
let tokenized = tokenizer.encode(text);
|
| 14 |
+
document.querySelector("#totalLength").textContent = `Total length: ${tokenized.ids.length}`;
|
| 15 |
let output = document.querySelector("#output");
|
| 16 |
output.contentDocument.body.innerHTML = "";
|
| 17 |
let colors = ["#ff0000", "#ff8700", "#ffd300", "#deff0a", "#a1ff0a", "#0aff99", "#0aefff", "#147df5", "#580aff", "#be0aff"].sort(() => Math.random() - 0.5);
|