Model 2.9: distilled from TypeSafe, no more false blocks on branded notices

.dockerignore

@@ -41,6 +41,7 @@ src/mBERT/training/model-training/dataset/
 !src/api_interface/models/
 !src/mBERT/training/model-training/mbert_ots_model_2.5.pth
 !src/mBERT/training/model-training/mbert_ots_model_2.7.pth
+!src/mBERT/training/model-training/mbert_ots_model_2.9.pth
 *_v1.*
 *_v2.0.*
 *_1.*
@@ -108,6 +109,7 @@ config/*.local.*
 !src/api_interface/
 !src/mBERT/training/model-training/mbert_ots_model_2.5.pth
 !src/mBERT/training/model-training/mbert_ots_model_2.7.pth
+!src/mBERT/training/model-training/mbert_ots_model_2.9.pth
 !src/mBERT/training/model-training/dataset/sms_spam_phishing_dataset_v2.4_combined.csv
 !requirements*.txt
 !start*.sh

.github/workflows/model-gate.yml

@@ -0,0 +1,55 @@
+name: Model gate
+
+# Scores the checkpoint that settings.py points at on the small eval sets and
+# fails the PR if it regresses past evals/gate_thresholds.json. Runs when a
+# checkpoint, the model settings, or the eval tooling changes.
+on:
+  pull_request:
+    paths:
+      - "src/mBERT/training/model-training/*.pth"
+      - "src/api_interface/config/settings.py"
+      - "evals/**"
+      - ".github/workflows/model-gate.yml"
+  workflow_dispatch:
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: false
+
+      - name: Find the default checkpoint
+        id: ckpt
+        run: |
+          p=$(grep -o 'mBERT/training/model-training/mbert_ots_model_[0-9.]*\.pth' src/api_interface/config/settings.py | head -1)
+          echo "path=src/$p" >> "$GITHUB_OUTPUT"
+          echo "oid=$(git show HEAD:src/$p | grep oid | cut -d: -f2)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache the checkpoint and vocab
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ steps.ckpt.outputs.path }}
+            evals/assets/bert-base-multilingual-cased-vocab.txt
+          key: lfs-${{ steps.ckpt.outputs.oid }}
+
+      - name: Pull the checkpoint from LFS
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: git lfs pull --include "${{ steps.ckpt.outputs.path }},evals/assets/bert-base-multilingual-cased-vocab.txt"
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install CPU torch and transformers
+        run: |
+          python -m pip install --upgrade pip
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+          pip install transformers
+
+      - name: Score and gate
+        run: python evals/gate.py --model "${{ steps.ckpt.outputs.path }}"

.github/workflows/release-image.yml

@@ -0,0 +1,58 @@
+name: Release image
+
+# Builds the Docker image for linux/amd64 and linux/arm64 and pushes it to
+# Docker Hub. Refuses to finish unless the pushed manifest lists both
+# platforms, so a single-architecture image can never reach the public host
+# again. Needs repository secrets DOCKERHUB_USERNAME and DOCKERHUB_TOKEN.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Image tag to publish (e.g. 2.9)"
+        required: true
+      latest:
+        description: "Also move the latest tag"
+        type: boolean
+        default: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Work out the tags
+        id: tags
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then tag="${GITHUB_REF_NAME#v}"; latest=true; else tag="${{ inputs.tag }}"; latest="${{ inputs.latest }}"; fi
+          t="telecomsxchange/opentextshield:$tag"
+          [ "$latest" = "true" ] && t="$t,telecomsxchange/opentextshield:latest"
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"; echo "list=$t" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.tags.outputs.list }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Verify both platforms are in the manifest
+        run: |
+          m=$(docker buildx imagetools inspect "telecomsxchange/opentextshield:${{ steps.tags.outputs.tag }}")
+          echo "$m" | grep -q "linux/amd64" || { echo "amd64 missing"; exit 1; }
+          echo "$m" | grep -q "linux/arm64" || { echo "arm64 missing"; exit 1; }
+          echo "manifest ok: amd64 + arm64"

README.md

@@ -5,7 +5,7 @@
 
 # OpenTextShield (OTS)
 
-**Open-source spam and phishing detection for SMS, in more than 100 languages.**
+**Open-source spam and phishing detection for SMS. Built on a 104-language model and trained on messages in 20 languages.**
 
 OpenTextShield is a compact classifier (a fine-tuned multilingual BERT, about 180M parameters) that labels a text message as `ham`, `spam` or `phishing` in around 150 ms on a small CPU instance. It runs on your own servers as a REST API, an SMPP proxy in front of your SMSC, or both. No third-party AI service is involved.
 

evals/bench_typesafe.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Score TypeSafe on evals/datasets/bench100_v1.csv with the labeling-guide prompt.
+
+Writes evals/results/predictions_bench100_v1_typesafe.json in the same shape
+as run_eval.py's dumps, so the same message set can be compared head to head.
+Needs TYPESAFE_API_KEY.
+"""
+import asyncio, csv, json, sys, time
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from label_audit import QUESTIONS, TYPESAFE_MODEL, REPO_ROOT  # noqa: E402
+
+BENCH = REPO_ROOT / "evals/datasets/bench100_v1.csv"
+OUT = REPO_ROOT / "evals/results/predictions_bench100_v1_typesafe.json"
+
+
+async def main():
+    from typesafe_sdk import AsyncTypeSafeClient, RetryPolicy
+    rows = list(csv.DictReader(open(BENCH, newline="", encoding="utf-8")))
+    out = []
+    async with AsyncTypeSafeClient(retry=RetryPolicy(max_retries=4, timeout=60.0)) as client:
+        for r in rows:
+            t0 = time.time()
+            resp = await client.system_one({"sms": r["text"]}, QUESTIONS, model=TYPESAFE_MODEL)
+            ms = (time.time() - t0) * 1000
+            ans = resp.answers["label"] if hasattr(resp, "answers") else resp["answers"]["label"]
+            choice = getattr(ans, "choice", None) or ans["choice"]
+            conf = getattr(ans, "confidence", None) or ans["confidence"]
+            out.append({"text": r["text"], "gold": r["label"], "pred": choice, "conf": round(float(conf), 4),
+                        "latency_ms": round(ms, 1)})
+    n = len(out); acc = sum(o["pred"] == o["gold"] for o in out) / n
+    json.dump({"model": TYPESAFE_MODEL, "accuracy": acc, "rows": out}, open(OUT, "w"), indent=1, ensure_ascii=False)
+    lat = sorted(o["latency_ms"] for o in out)
+    print(f"{TYPESAFE_MODEL}: accuracy {acc:.1%} on {n}, median latency {lat[n//2]:.0f} ms; wrote {OUT.relative_to(REPO_ROOT)}")
+    for o in out:
+        if o["pred"] != o["gold"]:
+            print(f"  MISS gold={o['gold']} pred={o['pred']} conf={o['conf']}: {o['text'][:90]}")
+
+asyncio.run(main())

evals/calibrate_thresholds.py

@@ -48,10 +48,13 @@
 from loaders import LOADERS, production_normalize  # noqa: E402
 
 
-def load_logits(model_path, samples, device="cpu", batch_size=32, normalize=True):
+DEVICE = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
+
+
+def load_logits(model_path, samples, device=DEVICE, batch_size=32, normalize=True):
     tok = BertTokenizerFast(vocab_file=str(VOCAB_FILE), do_lower_case=False)
     model = BertForSequenceClassification(BertConfig(vocab_size=119547, num_labels=3))
-    model.load_state_dict(torch.load(model_path, map_location=device, weights_only=True))
+    model.load_state_dict(torch.load(model_path, map_location="cpu", weights_only=True))
     model.eval().to(device)
     texts = [s["text"] for s in samples]
     if normalize:

evals/datasets/README.md

@@ -6,6 +6,7 @@
 | `fable5_adversarial_v1_clean.csv` | 71 | The same suite without rows that share a template with `synthetic_fable5_v1.csv`, which every model since 2.7 trains on. Use this split for claims about unseen messages. |
 | `fable5_template_twins.json` | 56 | Why each row was left out of the clean split: its closest training row and the probability that both come from the same template. |
 | `hard_legit_a2p_v1.csv` | 40 | Legitimate bank, delivery, billing, security and code messages that look like scams. Measures false blocks. **Handwritten synthetic examples**, not real traffic. |
+| `bench100_v1.csv` | 100 | Hand-written head-to-head set used for the TypeSafe comparison and the 2.9 distillation: 34 ham (17 non-English), 33 spam, 33 phishing, covering service notices, social engineering and prize/job lures. `evals/bench_typesafe.py` scores TypeSafe on it. |
 | `mishra_soni_5971.csv` | 5,971 | Public Mishra & Soni SMS phishing set. Its "smishing" label includes premium-rate prize spam, which `docs/LABELING_GUIDE.md` also calls phishing. |
 
 UCI SMS Spam and IMC25 are downloaded at eval time; see `evals/TIER1.md`.

evals/datasets/bench100_v1.csv

@@ -0,0 +1,101 @@
+id,text,label
+1,Hey are we still on for dinner tonight? I can pick you up at 7,ham
+2,Your Uber code is 4471. Never share this code with anyone.,ham
+3,Chase: Did you make a $184.20 purchase at TARGET on 09/18? Reply YES or NO.,ham
+4,"Mom I landed safely, will call you when I get to the hotel",ham
+5,Reminder: your dentist appointment with Dr. Patel is tomorrow at 10:30am. Reply C to confirm or R to reschedule.,ham
+6,Amazon: Your package with Bluetooth headphones was delivered. It was handed to a resident.,ham
+7,lol did you see what he posted in the group chat,ham
+8,Your Google verification code is 882913,ham
+9,HMRC will never contact you by text asking for bank details. Report suspicious messages to 60599.,ham
+10,"Running 15 min late, traffic on the bridge is terrible. Order me a flat white?",ham
+11,Delta: Flight DL1284 to Atlanta is now departing from gate B22. Boarding begins at 3:05pm.,ham
+12,"Can you send me the wifi password again, my phone forgot it",ham
+13,Your electricity bill of $92.41 is due on Sep 28. Pay in the ConEd app or at coned.com. Thank you.,ham
+14,"Happy birthday!!! Hope you have an amazing day, drinks on me this weekend",ham
+15,"Hola, ¿a qué hora llegas mañana? Te espero en la estación.",ham
+16,Votre code de connexion Doctolib est 371 905. Il expire dans 10 minutes.,ham
+17,"Dein DHL Paket wurde heute um 14:12 zugestellt. Vielen Dank, dass Sie DHL nutzen.",ham
+18,سأتأخر قليلاً عن الاجتماع، ابدؤوا بدوني وسألحق بكم,ham
+19,आज शाम को घर आ रहे हो? मम्मी ने खाना बनाया है,ham
+20,"Oi, chegou o boleto do condomínio? Preciso pagar até sexta.",ham
+21,"Your prescription is ready for pickup at CVS Pharmacy, 1200 Main St. Store hours 8am-10pm.",ham
+22,"Netflix: A new device signed in to your account from Chicago, IL. If this wasn't you, change your password in Account settings.",ham
+23,"Thanks for coming to the interview today. We'll be in touch by Friday with next steps. - Sarah, HR",ham
+24,Your table for 4 at Nobu is confirmed for Saturday 8:00pm. Reply CANCEL to cancel.,ham
+25,Bank of America: Your account ending 4432 has a low balance of $23.15. This is an automated alert you set up.,ham
+26,"just finished the report, sending it over now. let me know if anything looks off",ham
+27,Vodafone: You've used 80% of your 20GB monthly data. Your allowance resets on 1 Oct.,ham
+28,Practice is cancelled tonight because of the rain. Coach will send the new schedule tomorrow.,ham
+29,"Your Lyft driver Marcus is arriving in a gray Toyota Camry, plate 7XKL221.",ham
+30,Grandpa's surgery went well. He's resting now. Will update everyone tonight.,ham
+31,Verizon: Your bill of $118.60 is now available. View or pay at myvzw.com. Autopay will process on 9/30.,ham
+32,"Can u pls transfer me the £20 for the tickets when you get a sec, sorted it out with Jake already",ham
+33,Weather alert: Flash flood warning for Harris County until 9pm. Avoid low-lying roads.,ham
+34,"Your Airbnb host Maria sent you a message: Check-in is after 3pm, the lockbox code is in the app.",ham
+35,FLASH SALE! 50% off everything at FashionNova this weekend only. Shop now: fnova.co/sale,spam
+36,"Get a $5,000 personal loan approved in minutes, no credit check! Apply at quickcash-loans.net",spam
+37,Join our VIP crypto signals group and earn 30% monthly. Limited spots! t.me/cryptokingsignals,spam
+38,WIN BIG tonight at LuckySpin Casino! 200 free spins on signup. Play now luckyspin.bet,spam
+39,Tired of belly fat? Try KetoBurn and lose 10lbs in 2 weeks. Order today ketoburn-shop.com,spam
+40,"Earn $3,000/week working from home part-time! No experience needed. Start now: wfh-income.biz",spam
+41,Hot singles in your area are waiting to chat! Sign up free at meetlocal.xyz,spam
+42,"Vote for Councilman Reyes on Nov 5! Lower taxes, safer streets. Paid for by Reyes for Council.",spam
+43,Pizza Hut: 2 large pizzas for $15.99 this week only! Order online or call your local store.,spam
+44,Solar panels with $0 down! Cut your electric bill by 70%. Free quote at sunpower-deals.com,spam
+45,Extended car warranty! Your vehicle coverage may be expiring. Call 888-555-0142 to renew today.,spam
+46,"Unlock 10,000 followers overnight! Grow your Instagram fast. Try it: instaboost.pro",spam
+47,New iPhone 17 for only $99 with a 24-month plan at MobileMart! Visit your nearest store now.,spam
+48,"Bet £10 get £40 in free bets at BetKing. New customers only, 18+, T&Cs apply. betking.co.uk",spam
+49,"Make passive income with our forex robot, 95% win rate! Download at fxautobot.io",spam
+50,Sexy lingerie sale! Up to 70% off at LaceLux this week. lacelux.shop/sale,spam
+51,"Refinance your mortgage at 2.9% APR! Rates are dropping, lock yours in today at homeloanpro.us",spam
+52,"Cheap Viagra and Cialis online, no prescription needed. Discreet shipping. pharma-direct.ru",spam
+53,Become a distributor of NutriLife supplements and build your own business! Message JOIN to 55512,spam
+54,Black Friday came early! 80% off electronics at MegaDeals. Limited stock: megadeals.store,spam
+55,¡Gana dinero desde casa! Hasta €2000 al mes sin experiencia. Regístrate: trabajofacil.es,spam
+56,"Promo Bouygues : forfait 100Go à 9,99€/mois pendant 12 mois ! Souscrivez sur bouygues-promo.fr",spam
+57,Jetzt 100€ Bonus sichern! Online Casino mit über 2000 Spielen. spielhalle24.de,spam
+58,عرض خاص! خصم 40% على جميع العطور في متجر الياسمين هذا الأسبوع فقط. تسوق الآن yasmin-shop.com,spam
+59,Ganhe até R$500 por dia com apostas online! Cadastre-se grátis: apostaganha.bet,spam
+60,Best deals on flights to Dubai from $299! Book by Sunday at flyaway-travel.com,spam
+61,Ramen Republic is now open on 5th Ave! Show this text for a free appetizer with any entree.,spam
+62,Learn to code in 12 weeks and land a $100k job. Enroll in CodeCamp today: codecamp.edu/apply,spam
+63,Bitcoin is about to explode! Buy now before it's too late. Free guide: btc-profits.io,spam
+64,"Sell your house fast for cash! Any condition, close in 7 days. Call 305-555-0199",spam
+65,Reply STOP to opt out. GymFlex: Join today for $1 and get your first month free! gymflex.com/join,spam
+66,"Your favorite brands, 90% off! Designer bags, watches, sneakers. Shop luxreplica.cc",spam
+67,LAST CHANCE: Our Labor Day mattress sale ends tonight! Save up to $800 at SleepWell.,spam
+68,USPS: Your parcel could not be delivered because of an unpaid customs fee. Settle it within 24h to avoid return: http://usps-redelivery.top/pay,phishing
+69,Chase Alert: Your account has been locked due to suspicious activity. Verify your identity at chase-secure-login.com to restore access.,phishing
+70,Congratulations! You've been selected to receive a $1000 Walmart gift card. Claim here: walmart-rewards.win/claim,phishing
+71,"Hi Mum, I dropped my phone in the sink, this is my new number. Can you WhatsApp me on this number, I need help with something urgent x",phishing
+72,"IRS: You have a pending tax refund of $1,432.50. Submit your bank details to receive it: irs-refund-portal.com",phishing
+73,Netflix: Your payment failed and your membership will be cancelled. Update your billing info now at netflix-billing-update.com,phishing
+74,Apple ID: Your iCloud storage is full and your photos will be deleted. Upgrade now: icloud-storage-renew.net,phishing
+75,"Your Amazon order #114-9982 for a MacBook Pro ($2,399) has shipped. If you did not place this order, call 1-844-555-0177 immediately.",phishing
+76,"Hi, this is Jennifer from Amazon HR. You have been selected for a remote data entry job, $450/day. Reply YES to start.",phishing
+77,"Wells Fargo: A wire transfer of $4,800 is pending. If you did not authorize this, cancel here: wf-securecancel.com",phishing
+78,FedEx: Your package is on hold. A $1.99 redelivery fee is required. Pay at fedex-tracking-hold.info,phishing
+79,Your PayPal account has been limited. Confirm your identity within 48 hours to avoid permanent suspension: paypal-verify-id.com,phishing
+80,"Hey, is this still your number? I think we met at Lisa's party last month. Sorry if wrong number!",phishing
+81,Hey can you send me the 6 digit code you just got? I accidentally put your number in when logging in,phishing
+82,Coinbase: Unusual login from Russia detected. Secure your wallet immediately at coinbase-secure-auth.com or your funds may be lost.,phishing
+83,Final notice: Your toll balance of $12.51 is overdue. Pay now to avoid a $50 late fee: ezpass-tolls-pay.com,phishing
+84,Verizon: Your account will be suspended due to a billing error. Update your payment details at vzw-billing-fix.com,phishing
+85,Microsoft: Your Office 365 password expires today. Keep your current password by clicking: ms365-keep-password.com,phishing
+86,"Dear customer, your SSN has been suspended due to suspicious activity. Call 202-555-0163 immediately to avoid arrest.",phishing
+87,"You have won the UK National Lottery Mobile Draw, £850,000! Send your name, address and bank details to claim@uknl-draw.com",phishing
+88,Alerte CAF : un remboursement de 254€ vous attend. Confirmez vos coordonnées bancaires : caf-remboursement.fr,phishing
+89,Sparkasse: Ihr Konto wurde aus Sicherheitsgründen eingeschränkt. Bitte bestätigen Sie Ihre Daten unter sparkasse-sicherheit.de,phishing
+90,BBVA: Hemos detectado un acceso no autorizado. Verifique su cuenta ahora en bbva-seguridad-online.com o será bloqueada.,phishing
+91,بنك الراجحي: تم إيقاف بطاقتك مؤقتاً. لتفعيلها يرجى تحديث بياناتك عبر الرابط alrajhi-update.com,phishing
+92,SBI: Your KYC has expired. Your account will be blocked today. Update immediately at sbi-kyc-update.in,phishing
+93,"Correios: sua encomenda está retida na alfândega. Pague a taxa de R$9,90 para liberar: correios-liberacao.com",phishing
+94,Your Instagram account was reported for copyright violation. Appeal within 24h or it will be deleted: ig-copyright-appeal.com,phishing
+95,"Dad it's me, I'm in trouble and can't talk right now. Please send $500 to this Zelle: quickhelp99@gmail.com, I'll explain later",phishing
+96,DHL: Please confirm your address for the delivery scheduled today. Address confirmation: dhl-parcel-confirm.com,phishing
+97,Your Spotify Premium was charged $14.99 to a card ending 8821. Not you? Dispute here: spotify-dispute-center.com,phishing
+98,Binance customer support: Your withdrawal of 0.45 BTC is pending. Enter your 2FA code at binance-withdraw-confirm.com to complete it.,phishing
+99,"Selected candidate: TikTok is hiring part-time video likers, $200/day paid daily. WhatsApp +1 415 555 0134 to start.",phishing
+100,Your Medicare card needs to be replaced with a new chip card. Confirm your Medicare number here to avoid losing coverage: medicare-newcard.org,phishing