From 5b059ce6deb2b4132b445a0db3816388367acccf Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Mon, 11 Aug 2025 22:24:54 +0200 Subject: feat: auto-captions --- apps/transcription/README.md | 86 ++++++++++++++++++++++ apps/transcription/requirements.txt | 5 ++ apps/transcription/transcription.py | 143 ++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 apps/transcription/README.md create mode 100644 apps/transcription/requirements.txt create mode 100644 apps/transcription/transcription.py (limited to 'apps/transcription') diff --git a/apps/transcription/README.md b/apps/transcription/README.md new file mode 100644 index 00000000..07be6790 --- /dev/null +++ b/apps/transcription/README.md @@ -0,0 +1,86 @@ +Before you follow anything in this guide, please make sure you've followed the steps in the [README](../../README.md) (under "Optional: Auto-captions (Transcription) Setup"). + +Open your terminal and make sure you're in the `apps/transcription` directory. + +1. Create virtual environment + +```bash +python -m venv env +``` + +2. Activate it + +**Windows:** + +```bash +env\Scripts\activate +``` + +**macOS/Linux:** + +```bash +source env/bin/activate +``` + +> Note: if you're using VS Code/Cursor and you're seeing errors with the imports about the modules not being found, +> You might have to press CTRL + Shift + P -> Python: Select Interpreter -> Enter interpreter path -> Find -> env -> scripts -> python.exe + +3. Install libraries/packages/whatever you wanna call them + +```bash +pip install -r requirements.txt +``` + +4. Make sure you have a Modal account. If you don't: [create one](https://modal.com/) + +> If you don't know what Modal is: it allows us to process the actual audio and transcribe with Whisper by providing the infra to run Python code with a lot of RAM, generally affordable. + +5. Once you've got a Modal accoumt, run this: + +```bash +python -m modal setup +``` + +It's gonna open a browser so you can authenticate. + +6. Test it if you want to make sure it actually works: + +```bash +modal run transcription.py +``` + +6. Deploy the function! + +```bash +modal deploy transcription.py +``` + +7. Set the required secrets in Modal + +So the script we just deployed interacts with Cloudflare to do two things: + +- Download the audio (so it can be transcribed with Whisper) +- Delete the file after processing (privacy) + +To do those things, the script needs access to these environment variables: +```bash +CLOUDFLARE_ACCOUNT_ID=your-account-id +R2_ACCESS_KEY_ID=your-access-key-id +R2_SECRET_ACCESS_KEY=your-secret-access-key +R2_BUCKET_NAME=opencut-transcription +``` + +Remember, we set these earlier in `.env.local`. + +So let's do it: + + - Go to [Modal Secrets](https://modal.com/secrets/mazewinther/main) + - Click "Custom" and enter "opencut-r2-secrets" for the name. + - Now you can just click "Import .env" and copy/paste the 4 variables from your `.env.local` file. Copy and paste these only: + ```bash + CLOUDFLARE_ACCOUNT_ID=your-account-id + R2_ACCESS_KEY_ID=your-access-key-id + R2_SECRET_ACCESS_KEY=your-secret-access-key + R2_BUCKET_NAME=opencut-transcription + ``` + - Click "Done" and you should see some cool particles! \ No newline at end of file diff --git a/apps/transcription/requirements.txt b/apps/transcription/requirements.txt new file mode 100644 index 00000000..c002e683 --- /dev/null +++ b/apps/transcription/requirements.txt @@ -0,0 +1,5 @@ +modal +openai-whisper +boto3 +pydantic +cryptography \ No newline at end of file diff --git a/apps/transcription/transcription.py b/apps/transcription/transcription.py new file mode 100644 index 00000000..ad55f6d4 --- /dev/null +++ b/apps/transcription/transcription.py @@ -0,0 +1,143 @@ +import modal +from pydantic import BaseModel + +app = modal.App("opencut-transcription") + +class TranscribeRequest(BaseModel): + filename: str + language: str = "auto" + decryptionKey: str = None + iv: str = None + +@app.function( + image=modal.Image.debian_slim() + .apt_install(["ffmpeg"]) + .pip_install(["openai-whisper", "boto3", "fastapi[standard]", "pydantic", "cryptography"]), + gpu="A10G", + timeout=300, # 5m + secrets=[modal.Secret.from_name("opencut-r2-secrets")] +) +@modal.fastapi_endpoint(method="POST") +def transcribe_audio(request: TranscribeRequest): + import whisper + import boto3 + import tempfile + import os + import json + + try: + filename = request.filename + language = request.language + decryption_key = request.decryptionKey + iv = request.iv + + if not filename: + return { + "error": "Missing filename parameter" + } + + # Initialize R2 client + s3_client = boto3.client( + 's3', + endpoint_url=f'https://{os.environ["CLOUDFLARE_ACCOUNT_ID"]}.r2.cloudflarestorage.com', + aws_access_key_id=os.environ["R2_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"], + region_name='auto' + ) + + # Create temporary file for audio + with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_file: + temp_path = temp_file.name + + try: + # Download audio from R2 + s3_client.download_file( + os.environ["R2_BUCKET_NAME"], + filename, + temp_path + ) + + # If decryption key provided, decrypt the file directly (zero-knowledge) + if decryption_key and iv: + import base64 + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + from cryptography.hazmat.backends import default_backend + + # Read the encrypted file + with open(temp_path, 'rb') as f: + encrypted_data = f.read() + + # Decode the key and IV from base64 + key_bytes = base64.b64decode(decryption_key) + iv_bytes = base64.b64decode(iv) + + # Decrypt the data using AES-GCM + # Extract the tag (last 16 bytes) and ciphertext + tag = encrypted_data[-16:] + ciphertext = encrypted_data[:-16] + + cipher = Cipher( + algorithms.AES(key_bytes), + modes.GCM(iv_bytes, tag), + backend=default_backend() + ) + decryptor = cipher.decryptor() + decrypted_data = decryptor.update(ciphertext) + decryptor.finalize() + + # Write decrypted audio back to temp file + with open(temp_path, 'wb') as f: + f.write(decrypted_data) + + # Load Whisper model + model = whisper.load_model("base") + + # Transcribe audio + if language == "auto": + result = model.transcribe(temp_path) + else: + result = model.transcribe(temp_path, language=language.lower()) + + # Delete audio file from R2 (cleanup) + s3_client.delete_object( + Bucket=os.environ["R2_BUCKET_NAME"], + Key=filename + ) + + # Adjust segment timing - Whisper is consistently late by ~500ms + adjusted_segments = [] + for segment in result["segments"]: + adjusted_segment = segment.copy() + # Shift start/end times earlier by 500ms, don't go below 0 + adjusted_segment["start"] = max(0, segment["start"] - 0.5) + adjusted_segment["end"] = max(0.5, segment["end"] - 0.5) # Ensure duration is at least 0.5s + adjusted_segments.append(adjusted_segment) + + return { + "text": result["text"], + "segments": adjusted_segments, + "language": result["language"] + } + + finally: + # Clean up temporary file + if os.path.exists(temp_path): + os.unlink(temp_path) + + except Exception as e: + import traceback + print(f"Transcription error: {str(e)}") + print(f"Traceback: {traceback.format_exc()}") + + # Return error response that matches expected format + return { + "error": str(e), + "text": "", + "segments": [], + "language": "unknown" + } + +@app.local_entrypoint() +def main(): + # Test function - you can call this with modal run transcription.py + print("Transcription service is ready to deploy!") + print("Deploy with: modal deploy transcription.py") \ No newline at end of file -- cgit v1.3.1