Main

.gitignore

@@ -169,3 +169,7 @@ ml/tests/sample_audio/
 pytest_cache/
 .coverage
 htmlcov/
+
+audios/
+
+models/

ml_v2/data_structure.txt

@@ -0,0 +1,45 @@
+## Data Structures for Frontend <-> ML Backend Communication
+
+### Frontend to ML Backend (`/process` endpoint)
+
+When the frontend sends an audio file for processing, it makes a POST request to the `/process` endpoint of the `ml` backend.
+
+**Request Body:**
+```json
+{
+  "file_url": "https://your-azure-blob-storage-url/audio.webm",
+  "lat": 12.9716,
+  "lng": 77.5946
+}
+```
+
+- `file_url` (string): The URL of the audio file stored in Azure Blob Storage.
+- `lat` (number): The latitude of the user's location.
+- `lng` (number): The longitude of the user's location.
+
+---
+
+### ML Backend to Frontend
+
+After processing the audio, the `ml` backend returns a JSON object with the analysis results.
+
+**Response Body:**
+```json
+{
+  "language": "en",
+  "confidence": 0.98,
+  "transcript": "A sample transcription of the audio.",
+  "cluster_id": 12,
+  "embedding": [0.123, 0.456, ...],
+  "lat": 12.9716,
+  "lng": 77.5946
+}
+```
+
+- `language` (string): The detected language code (e.g., "en", "hi").
+- `confidence` (float): A score from 0.0 to 1.0 indicating the model's confidence in the language detection.
+- `transcript` (string): The text transcribed from the audio.
+- `cluster_id` (integer | null): The ID of the cluster the audio is assigned to if it's an unknown dialect. It is `null` for known languages.
+- `embedding` (array of floats): The vector representation of the audio.
+- `lat` (number): The latitude passed in the original request.
+- `lng` (number): The longitude passed in the original request.

ml_v2/encoder/main.py

@@ -3,6 +3,7 @@
 import logging
 import librosa
 from services.preprocess_audio import preprocess_audio
+from services.generate_embeddings import extract_embedding
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("encoder-service")
@@ -20,15 +21,15 @@ async def vectorize_audio(file: UploadFile = File(...)):
 
     audio = preprocess_audio(file)
 
-    logger.info("Generated embedding successfully")
+    logger.info("Preprocessed audio successfully")
 
+    embedding = extract_embedding(audio)
 
-    #just dummy data for now, will add the encoder tomorrow
-    vector = [1,2,3,4,5]
+    logger.info("Generated embedding successfully")
 
     return {
         "fileName": file.filename,
-        "embedding": vector
+        "embedding": embedding.tolist()
     }
 
 

ml_v2/encoder/services/generate_embeddings.py

@@ -0,0 +1,42 @@
+import torch 
+import librosa
+from functools import lru_cache
+from transformers import Wav2Vec2FeatureExtractor, Wav2Vec2Model
+from pathlib import Path
+import numpy
+
+SAMPLE_RATE = 16000
+
+@lru_cache()
+def load_embedding_model(model_name="facebook/wav2vec2-large-xlsr-53"):
+    base_path = Path(__file__).resolve().parent.parent /"models"/"wav2vec2"
+
+    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
+        model_name,
+        cache_dir = str(base_path)
+    )
+
+    model = Wav2Vec2Model.from_pretrained(
+        model_name,
+        cache_dir = str(base_path)
+    )
+
+    model.eval()
+
+    return model, feature_extractor
+
+def extract_embedding(audio):
+    model, feature_extractor = load_embedding_model()
+
+    inputs = feature_extractor(
+        audio, 
+        sampling_rate = SAMPLE_RATE,
+        return_tensors = "pt",
+        padding = True
+    )
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        embedding = outputs.last_hidden_state.mean(dim=1)
+
+    return embedding.squeeze().cpu().numpy()

ml_v2/encoder/services/preprocess_audio.py

@@ -5,11 +5,12 @@
 import os
 import shutil
 
+
 def preprocess_audio(file: UploadFile):
     SAMPLE_RATE = 16000
     filename = file.filename or "audio.wav"
-    file_ext = os.path.splittext(filename)[1] 
-
+    file_ext = os.path.splitext(filename)[1] 
+    
     try:
 
         with tempfile.NamedTemporaryFile(suffix=file_ext) as tmp:

ml_v2/orchastrater/pyproject.toml

@@ -8,6 +8,7 @@ dependencies = [
     "fastapi>=0.128.0",
     "faster-whisper>=1.2.1",
     "numpy>=2.4.1",
+    "psycopg2-binary>=2.9.11",
     "pydantic>=2.12.5",
     "requests>=2.32.5",
     "uvicorn>=0.40.0",

ml_v2/orchastrater/services/remote_encoder.py

@@ -2,11 +2,10 @@
 import requests
 import os
 from utils.logger import get_logger
+from utils.env import ENCODER_URL
 
 logger = get_logger(__name__)
-
-ENCODER_URL = "http://localhost:8001"  
-
+
 def get_audio_embedding(file_path: str):
     """
     Sends an audio file to the Encoder service and returns the vector.

ml_v2/orchastrater/services/whisper_utils.py

@@ -0,0 +1,50 @@
+from faster_whisper import WhisperModel
+from utils.logger import get_logger
+import os
+
+logger = get_logger("whisper-app")
+
+MODEL_SIZE = os.getenv("WHISPER_MODEL_SIZE","small")
+DEVICE = "cpu"
+COMPUTE_TYPE = "int8"
+
+class WhisperService:
+    _instance = None 
+
+    @classmethod
+    def get_instance(cls):
+
+        if cls._instance is None:
+            logger.info(f"Loading whisper model '{MODEL_SIZE}' on {DEVICE}")
+            cls._instance = WhisperModel(MODEL_SIZE,device=DEVICE,compute_type=COMPUTE_TYPE)
+            logger.info(f"Loaded whisper model successfully")
+        return cls._instance
+
+def transcribe_audio(file_path: str):
+    model = WhisperService.get_instance()
+
+    try:
+
+        segments, info = model.transcribe(
+            file_path,
+            beam_size = 5,
+            vad_filter = True,
+            vad_parameters = dict(min_silence_duration_ms = 500)
+        )
+
+        full_text = " ".join([segment.text for segment in segments]).strip()
+
+        result = {
+            "text" : full_text,
+            "language" : info.language,
+            "probability" : info.language_probability
+        }
+
+        logger.info(f"Transcribed audio successfully: {info.language} ({info.language_probability:.2f})")
+
+        return result
+
+    except Exception as e:
+        logger.error(f"Failed to transcribe audio: {e}")
+        raise
+

ml_v2/orchastrater/test_driver.py

@@ -1,18 +1,18 @@
 # orchestrator/test_driver.py
 import sys
+import os
 # Fix python path to find 'services'
 sys.path.append(".") 
 
 from services.remote_encoder import get_audio_embedding
-
-# Make a dummy file to test
-with open("test_audio.txt", "w") as f:
-    f.write("This is fake audio data")
+from services.whisper_utils import transcribe_audio
 
 try:
     print("Attempting to talk to Encoder...")
-    vector = get_audio_embedding("test_audio.txt")
+    # vector = get_audio_embedding("../audios/English.m4a")
+    result = transcribe_audio("../audios/English.m4a")
     print("\nSUCCESS! Received Vector:")
-    print(vector)
+    # print(vector)
+    print(result)
 except Exception as e:
     print(f"\nFAILED: {e}")

ml_v2/orchastrater/utils/db.py

@@ -0,0 +1,48 @@
+import psycopg2
+from psycopg2.extras import RealDictCursor
+from utils.logger import get_logger
+from utils.env import DATABASE_URL
+
+logger = get_logger("db")
+
+def get_db_connection():
+
+    try:
+        conn = psycopg2.connect(DATABASE_URL)
+        return conn
+    except Exception as e:
+        logger.error("Failed to connect to the db")
+        raise
+
+def excecute_query(query: str, params: tuple = None, fetch_one = False, fetch_all = False):
+
+    conn = None
+
+    try:
+        conn = get_db_connection()
+
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(query, params)
+
+            if fetch_one:
+                result = cur.fetch_one()
+
+            elif fetch_all:
+                result = cur.fetch_all()
+
+            else:
+                result = None
+
+            conn.commit()
+
+            return result
+
+    except Exception as e:
+        if conn: 
+            conn.rollback()
+        logger.error(f"DB Query Failed: {e} | Query: {query}")
+        raise
+
+    finally: 
+        if conn:
+            conn.close()

ml_v2/orchastrater/utils/env.py

@@ -0,0 +1,17 @@
+import os
+import sys
+from dotenv import load_dotenv
+
+# Load .env from the current directory
+load_dotenv()
+
+def get_env_variable(key: str, default: str = None, required: bool = False) -> str:
+    value = os.getenv(key, default)
+    if required and not value:
+        print(f"CRITICAL ERROR: Missing environment variable '{key}'")
+        sys.exit(1)
+    return value
+
+# Config Variables
+DATABASE_URL = get_env_variable("DATABASE_URL", required=True)
+ENCODER_URL = get_env_variable("ENCODER_URL", "http://localhost:8001")