commit 5e8cf7790ca52ac0013344d460e709ada08279d1
parent f5b0bb63ae37b1efb96c0edc85f8fabbca83355d
Author: MTRNord <mtrnord1@gmail.com>
Date: Wed, 28 Sep 2022 18:29:03 +0200
Add new sample of spam and make sure that data send for review is sanitized
Diffstat:
5 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,5 @@
/hypertuner_logs
/target
/logs
-/wandb
-\ No newline at end of file
+/wandb
+/data
+\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
@@ -30,6 +30,15 @@ dependencies = [
]
[[package]]
+name = "aho-corasick"
+version = "0.7.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "ansi_term"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -575,6 +584,15 @@ dependencies = [
]
[[package]]
+name = "linkify"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96dd5884008358112bc66093362197c7248ece00d46624e2cf71e50029f8cff5"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
name = "lock_api"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -648,6 +666,7 @@ dependencies = [
"axum",
"axum-auth",
"color-eyre",
+ "linkify",
"once_cell",
"schemars",
"serde",
@@ -656,6 +675,7 @@ dependencies = [
"tokio",
"tracing",
"tracing-subscriber",
+ "voca_rs",
]
[[package]]
@@ -876,6 +896,8 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
dependencies = [
+ "aho-corasick",
+ "memchr",
"regex-syntax",
]
@@ -1069,6 +1091,16 @@ dependencies = [
]
[[package]]
+name = "stfu8"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "019f0c664fd85d5a87dcfb62b40b691055392a35a6e59f4df83d4b770db7e876"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
+[[package]]
name = "subtle"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1342,6 +1374,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
[[package]]
+name = "unicode-segmentation"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
+
+[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1360,6 +1398,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
+name = "voca_rs"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4646f8794c283a89bfc3bc1a49f99562163fc8d01b2771bcf9c844e9a6f38d94"
+dependencies = [
+ "regex",
+ "stfu8",
+ "unicode-segmentation",
+]
+
+[[package]]
name = "want"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/crates/model_server/Cargo.toml b/crates/model_server/Cargo.toml
@@ -10,6 +10,7 @@ version = "0.1.0"
axum = "0.5.16"
axum-auth = {version = "0.3", default-features = false, features = ["auth-bearer"]}
color-eyre = "0.6.2"
+linkify = "0.9.0"
once_cell = "1.15.0"
serde = {version = "1.0", features = ["derive"]}
serde_json = "1.0"
@@ -17,6 +18,7 @@ tensorflow = {version = "0.19.1", features = ["tensorflow_gpu"]}
tokio = {version = "1.0", features = ["full"]}
tracing = "0.1"
tracing-subscriber = {version = "0.3", features = ["env-filter"]}
+voca_rs = "1.14.0"
[dev-dependencies]
schemars = "0.8.10"
diff --git a/crates/model_server/src/main.rs b/crates/model_server/src/main.rs
@@ -1,12 +1,14 @@
use axum::{http::StatusCode, response::IntoResponse, routing::post, Json, Router};
use axum_auth::AuthBearer;
use color_eyre::eyre::{bail, Result};
+use linkify::LinkFinder;
use once_cell::sync::OnceCell;
use serde::{Deserialize, Serialize};
use std::io::Write;
use std::{fs::OpenOptions, net::SocketAddr};
use tensorflow::{Graph, SavedModelBundle, SessionOptions, SessionRunArgs, Tensor};
use tracing::{debug, error, info};
+use voca_rs::strip;
static GRAPH: OnceCell<Graph> = OnceCell::new();
static MODEL: OnceCell<SavedModelBundle> = OnceCell::new();
@@ -126,9 +128,21 @@ async fn submit_for_review(
.append(true)
.create(true)
.open("./data/review.txt");
+
+ // Sanitize
+ // We remove newlines, html tags and links
+ let sanitized = payload.input_data.replace(['\r', '\n'], " ");
+ let mut sanitized = strip::strip_tags(&sanitized);
+ let mut finder = LinkFinder::new();
+ let cloned_sanitized = sanitized.clone();
+ finder.url_must_have_scheme(false);
+ let links: Vec<_> = finder.links(&cloned_sanitized).collect();
+ for link in links {
+ sanitized = sanitized.replace(link.as_str(), " ");
+ }
match file {
Ok(mut file) => {
- if let Err(e) = writeln!(file, "{}", payload.input_data) {
+ if let Err(e) = writeln!(file, "{}", sanitized) {
eprintln!("Couldn't write to file: {}", e);
return StatusCode::INTERNAL_SERVER_ERROR;
}
diff --git a/input/MatrixData b/input/MatrixData
@@ -5597,4 +5597,5 @@ ham I'm writing a bot. I want to be able to provide the bot with the cross signi
ham O thanks, i'll take a look
ham Hello all you Element experts, I'm setting up a homelab with Synapse and Element-web. I have read that there are security downsides in hosting both on the same domain. Can someone confirm whether it is still true, even on different subdomains ?
ham Greg, can you call me back once you get this?
-spam You have won a million dollars! Fill out your bank details here...
-\ No newline at end of file
+spam You have won a million dollars! Fill out your bank details here...
+spam 💨Don’t miss this chance to be Rich ✌️👍🙏DiscJockey is mine name and am an admin who has his own store link and group link That’s my group link , you can check out my legit works there and proof 🙏UPDATE Services I offer: Sell PayPal account verified – PayPal transfer *Sell Bank Transfer – Bank Login *Sell Clone card - Secured shipping tunnel *Sell cc Fullz & random Infos – 99% valid cards *Sell Dumps with pin track 1 and 2 101 201 *Sell Western Union money transfer services *Sell Gift Card Itune – Amazon – Ebay Clone/Credit Cards *Sell Booking flight ticket services – worldwide *Sell Electronics carding services *Sell SMTP - Pass Mail - PHP - RDP !! Follow Group Stage and contact me if of you are interested to make money ! Let’s make more bread ! Follow the rules ! Order or purchase and get instantly ✌️Partnership member needed for a long term business 🙏100%. 100% High quality Good and High Quality Dumps+Pins 90% Approval TRANSFER SERVICE BTC/USDT /CASHAPP TN DUA PAYMENT ARIZONA DUA MASS UI DUA has EDD RELOAD ALL STATE DUA PAYMENT METHOD IS AVAILABLE !! FULLZ PAYPAL CASH APP DUMPS+PINS FULLZ + DUMPS PAYPAL Transfer CashaPP transfer BANK LOG and bank 🏦deposit DUMP + P!N PRODUCTS CVV+SSN/D WU TRANSFER ♛ CARDING ALL APPLE ♛ With good product of good sell QUALITY STUFFS ! With a lot of customers worldwide and also a Valid Vendor 🇺🇸 BESTBUY TAP & PAY WORKING BINS VERIZON TAP & PAY WALMART - TAP & Pay Bin’s Amazon prime Tap & pay ••Cool random prices Random Bin’s •• COUNTRIES ! Verizon - AMAZON DUMPS with pin AMAZON TRACK1-2 Verizon- Track2- 1 With Pins Walmart - Track1-2 Pins ZIPCODE + PINS FULLZ + Ssn - details FULLZ - Dob + Pins Cvv - ZIPCODE + ATM Anyone who is ready and willing to follow the rules should immediately message me or contact me ! Telegram ✅
