feat: auto-captions

3 files changed, 234 insertions, 0 deletions

diff --git a/apps/transcription/README.md b/apps/transcription/README.md
new file mode 100644
index 00000000..07be6790
--- /dev/null
+++ b/apps/transcription/README.md
@@ -0,0 +1,86 @@
+Before you follow anything in this guide, please make sure you've followed the steps in the [README](../../README.md) (under "Optional: Auto-captions (Transcription) Setup").

+

+Open your terminal and make sure you're in the `apps/transcription` directory.

+

+1. Create virtual environment

+

+```bash

+python -m venv env

+```

+

+2. Activate it

+

+**Windows:**

+

+```bash

+env\Scripts\activate

+```

+

+**macOS/Linux:**

+

+```bash

+source env/bin/activate

+```

+

+> Note: if you're using VS Code/Cursor and you're seeing errors with the imports about the modules not being found,

+> You might have to press CTRL + Shift + P -> Python: Select Interpreter -> Enter interpreter path -> Find -> env -> scripts -> python.exe

+

+3. Install libraries/packages/whatever you wanna call them

+

+```bash

+pip install -r requirements.txt

+```

+

+4. Make sure you have a Modal account. If you don't: [create one](https://modal.com/)

+

+> If you don't know what Modal is: it allows us to process the actual audio and transcribe with Whisper by providing the infra to run Python code with a lot of RAM, generally affordable.

+

+5. Once you've got a Modal accoumt, run this:

+

+```bash

+python -m modal setup

+```

+

+It's gonna open a browser so you can authenticate.

+

+6. Test it if you want to make sure it actually works:

+

+```bash

+modal run transcription.py

+```

+

+6. Deploy the function!

+

+```bash

+modal deploy transcription.py

+```

+

+7. Set the required secrets in Modal

+

+So the script we just deployed interacts with Cloudflare to do two things:

+

+- Download the audio (so it can be transcribed with Whisper)

+- Delete the file after processing (privacy)

+

+To do those things, the script needs access to these environment variables:

+```bash

+CLOUDFLARE_ACCOUNT_ID=your-account-id

+R2_ACCESS_KEY_ID=your-access-key-id

+R2_SECRET_ACCESS_KEY=your-secret-access-key

+R2_BUCKET_NAME=opencut-transcription

+```

+

+Remember, we set these earlier in `.env.local`.

+

+So let's do it:

+

+   - Go to [Modal Secrets](https://modal.com/secrets/mazewinther/main)

+   - Click "Custom" and enter "opencut-r2-secrets" for the name.

+   - Now you can just click "Import .env" and copy/paste the 4 variables from your `.env.local` file. Copy and paste these only:

+      ```bash

+      CLOUDFLARE_ACCOUNT_ID=your-account-id

+      R2_ACCESS_KEY_ID=your-access-key-id

+      R2_SECRET_ACCESS_KEY=your-secret-access-key

+      R2_BUCKET_NAME=opencut-transcription

+      ```

+    - Click "Done" and you should see some cool particles!
\ No newline at end of file
diff --git a/apps/transcription/requirements.txt b/apps/transcription/requirements.txt
new file mode 100644
index 00000000..c002e683
--- /dev/null
+++ b/apps/transcription/requirements.txt
@@ -0,0 +1,5 @@
+modal

+openai-whisper

+boto3

+pydantic

+cryptography
\ No newline at end of file
diff --git a/apps/transcription/transcription.py b/apps/transcription/transcription.py
new file mode 100644
index 00000000..ad55f6d4
--- /dev/null
+++ b/apps/transcription/transcription.py
@@ -0,0 +1,143 @@
+import modal

+from pydantic import BaseModel

+

+app = modal.App("opencut-transcription")

+

+class TranscribeRequest(BaseModel):

+    filename: str

+    language: str = "auto"

+    decryptionKey: str = None

+    iv: str = None

+

+@app.function(

+    image=modal.Image.debian_slim()

+        .apt_install(["ffmpeg"])

+        .pip_install(["openai-whisper", "boto3", "fastapi[standard]", "pydantic", "cryptography"]),

+    gpu="A10G",

+    timeout=300, # 5m

+    secrets=[modal.Secret.from_name("opencut-r2-secrets")]

+)

+@modal.fastapi_endpoint(method="POST")

+def transcribe_audio(request: TranscribeRequest):

+    import whisper

+    import boto3

+    import tempfile

+    import os

+    import json

+    

+    try:

+        filename = request.filename

+        language = request.language

+        decryption_key = request.decryptionKey

+        iv = request.iv

+        

+        if not filename:

+            return {

+                "error": "Missing filename parameter"

+            }

+        

+        # Initialize R2 client

+        s3_client = boto3.client(

+            's3',

+            endpoint_url=f'https://{os.environ["CLOUDFLARE_ACCOUNT_ID"]}.r2.cloudflarestorage.com',

+            aws_access_key_id=os.environ["R2_ACCESS_KEY_ID"],

+            aws_secret_access_key=os.environ["R2_SECRET_ACCESS_KEY"],

+            region_name='auto'

+        )

+        

+        # Create temporary file for audio

+        with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_file:

+            temp_path = temp_file.name

+            

+            try:

+                # Download audio from R2

+                s3_client.download_file(

+                    os.environ["R2_BUCKET_NAME"], 

+                    filename, 

+                    temp_path

+                )

+                

+                # If decryption key provided, decrypt the file directly (zero-knowledge)

+                if decryption_key and iv:

+                    import base64

+                    from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes

+                    from cryptography.hazmat.backends import default_backend

+                    

+                    # Read the encrypted file

+                    with open(temp_path, 'rb') as f:

+                        encrypted_data = f.read()

+                    

+                    # Decode the key and IV from base64

+                    key_bytes = base64.b64decode(decryption_key)

+                    iv_bytes = base64.b64decode(iv)

+                    

+                    # Decrypt the data using AES-GCM

+                    # Extract the tag (last 16 bytes) and ciphertext

+                    tag = encrypted_data[-16:]

+                    ciphertext = encrypted_data[:-16]

+                    

+                    cipher = Cipher(

+                        algorithms.AES(key_bytes),

+                        modes.GCM(iv_bytes, tag),

+                        backend=default_backend()

+                    )

+                    decryptor = cipher.decryptor()

+                    decrypted_data = decryptor.update(ciphertext) + decryptor.finalize()

+                    

+                    # Write decrypted audio back to temp file

+                    with open(temp_path, 'wb') as f:

+                        f.write(decrypted_data)

+                

+                # Load Whisper model

+                model = whisper.load_model("base")

+                

+                # Transcribe audio

+                if language == "auto":

+                    result = model.transcribe(temp_path)

+                else:

+                    result = model.transcribe(temp_path, language=language.lower())

+                

+                # Delete audio file from R2 (cleanup)

+                s3_client.delete_object(

+                    Bucket=os.environ["R2_BUCKET_NAME"],

+                    Key=filename

+                )

+                

+                # Adjust segment timing - Whisper is consistently late by ~500ms

+                adjusted_segments = []

+                for segment in result["segments"]:

+                    adjusted_segment = segment.copy()

+                    # Shift start/end times earlier by 500ms, don't go below 0

+                    adjusted_segment["start"] = max(0, segment["start"] - 0.5)

+                    adjusted_segment["end"] = max(0.5, segment["end"] - 0.5)  # Ensure duration is at least 0.5s

+                    adjusted_segments.append(adjusted_segment)

+                

+                return {

+                    "text": result["text"],

+                    "segments": adjusted_segments,

+                    "language": result["language"]

+                }

+                

+            finally:

+                # Clean up temporary file

+                if os.path.exists(temp_path):

+                    os.unlink(temp_path)

+                    

+    except Exception as e:

+        import traceback

+        print(f"Transcription error: {str(e)}")

+        print(f"Traceback: {traceback.format_exc()}")

+        

+        # Return error response that matches expected format

+        return {

+            "error": str(e),

+            "text": "",

+            "segments": [],

+            "language": "unknown"

+        }

+

+@app.local_entrypoint()

+def main():

+    # Test function - you can call this with modal run transcription.py

+    print("Transcription service is ready to deploy!")

+    print("Deploy with: modal deploy transcription.py")
\ No newline at end of file