#!/usr/bin/env python3
"""kolwrite.py: transcribe audio and video with the KolWrite API from one file.

Python 3.9+, standard library only. Reads the API key from KOLWRITE_API_KEY and
never prints it.

  kolwrite.py transcribe meeting.mp3 --model oracle --speakers -o meeting.json --format json
  kolwrite.py transcribe https://example.com/talk.mp4 --format srt -o talk.srt
  kolwrite.py transcribe interview.m4a --language he --translate-to en
  kolwrite.py job <jobId>          status or result of a job started earlier
  kolwrite.py models               models and prices
  kolwrite.py languages            language codes

Exit codes: 0 done, 1 API or network error, 2 usage or missing key,
3 still running when --timeout passed (the job keeps running; check it with
"kolwrite.py job <jobId>"), 4 the job FAILED.

Source: https://console.kolwrite.com/cli/kolwrite.py (SHA-256 in kolwrite.py.sha256 beside it).
Docs: https://console.kolwrite.com/llms-full.txt
"""

import argparse
import base64
import json
import mimetypes
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid

VERSION = "1.0.0"
API_BASE = "https://app.kolwrite.com"
CONSOLE = "https://console.kolwrite.com"
INLINE_MAX_BYTES = 4 * 1024 * 1024
DIRECT_MAX_BYTES = 100 * 1024 * 1024
USER_AGENT = "kolwrite.py/" + VERSION

# Credits per audio hour; 1 credit = US$1.20. Generated from the console's site data
# (sites/kolwrite-console/scripts/seo/site.mjs); a test fails when they drift.
CREDIT_USD = 1.2
MODELS = [
    # id, name, credits per audio hour, translates, current
    ("base", "Base", 0.4, False, True),
    ("oracle_mini", "Oracle Mini", 0.8, True, True),
    ("oracle", "Oracle", 1.2, True, True),
    ("oracle_pro", "Oracle Pro", 1.4, True, True),
    ("oracle_max", "Oracle Max", 3.0, True, True),
    ("core", "Core (legacy)", 1.0, False, False),
    ("edge", "Edge (legacy)", 1.2, False, False),
]
DIARIZATION_CREDITS = 0.1
PRO_DIARIZATION_CREDITS = 0.3

CONTENT_TYPES = {
    ".mp3": "audio/mpeg", ".wav": "audio/wav", ".m4a": "audio/mp4", ".aac": "audio/aac", ".flac": "audio/flac",
    ".ogg": "audio/ogg", ".opus": "audio/ogg", ".webm": "video/webm", ".mp4": "video/mp4", ".mov": "video/quicktime",
    ".mkv": "video/x-matroska", ".avi": "video/x-msvideo", ".aif": "audio/aiff", ".aiff": "audio/aiff", ".wmv": "video/x-ms-wmv",
}

LANGUAGES_URL = CONSOLE + "/llms/languages.md"

EXIT_OK, EXIT_ERROR, EXIT_USAGE, EXIT_TIMEOUT, EXIT_FAILED = 0, 1, 2, 3, 4


class ApiError(Exception):
    def __init__(self, message, status=None, retryable=False):
        Exception.__init__(self, message)
        self.status = status
        self.retryable = retryable


def say(message):
    sys.stderr.write(message + "\n")
    sys.stderr.flush()


class Client:
    def __init__(self, key, base, sleep=time.sleep, retries=4):
        self.key = key
        self.base = base.rstrip("/")
        self.sleep = sleep
        self.retries = retries

    def redact(self, text):
        return str(text).replace(self.key, "[redacted]") if self.key else str(text)

    def _send(self, method, url, body=None, headers=None, timeout=60):
        request = urllib.request.Request(url, data=body, method=method, headers=headers or {})
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                return response.status, response.read()
        except urllib.error.HTTPError as error:
            return error.code, error.read()
        except (urllib.error.URLError, OSError) as error:
            reason = getattr(error, "reason", error)
            raise ApiError(self.redact("Could not reach KolWrite: %s" % reason), retryable=True)

    def call(self, method, path, payload=None, retry=True):
        """JSON request to the API. Retries throttling, 5xx and network errors when retry is True."""
        headers = {"x-api-key": self.key, "Accept": "application/json", "User-Agent": USER_AGENT}
        body = None
        if payload is not None:
            body = json.dumps(payload).encode("utf-8")
            headers["Content-Type"] = "application/json"
        attempt = 0
        while True:
            try:
                status, raw = self._send(method, self.base + path, body, headers)
                if 200 <= status < 300:
                    try:
                        return json.loads(raw.decode("utf-8") or "{}")
                    except ValueError:
                        raise ApiError("KolWrite returned an unreadable response (HTTP %d)." % status, status)
                raise api_error(status, raw, self.redact)
            except ApiError as error:
                if not (retry and error.retryable) or attempt >= self.retries:
                    raise
                attempt += 1
                wait = 2 ** attempt
                say("KolWrite: %s; retrying in %d s" % (error, wait))
                self.sleep(wait)

    def upload(self, path):
        """Local file -> fileUrl, inline up to 4 MiB, through the signed storage form up to 100 MiB."""
        size = os.path.getsize(path)
        if size == 0:
            raise ApiError("The file is empty: %s" % path)
        if size > DIRECT_MAX_BYTES:
            raise ApiError(
                "The file is %.1f MiB; uploads are limited to 100 MiB. Extract the audio first, for example:\n"
                "  ffmpeg -i %s -vn -ac 1 -ar 16000 -b:a 48k audio.mp3" % (size / 1048576.0, path))
        name = os.path.basename(path)
        extension = os.path.splitext(name)[1].lower()
        content_type = CONTENT_TYPES.get(extension) or mimetypes.guess_type(name)[0] or "application/octet-stream"
        with open(path, "rb") as handle:
            data = handle.read()
        if size <= INLINE_MAX_BYTES:
            stored = self.call("POST", "/upload", {"filename": name, "contentType": content_type,
                                                    "content": base64.b64encode(data).decode("ascii")})
            return stored["fileUrl"]
        form = self.call("POST", "/upload", {"filename": name, "contentType": content_type})
        for field in ("uploadUrl", "uploadFields", "fileUrl"):
            if field not in form:
                raise ApiError("KolWrite did not return a signed upload form (no %s)." % field)
        body, boundary = multipart(form["uploadFields"], name, content_type, data)
        # The signed storage form takes no API key.
        headers = {"Content-Type": "multipart/form-data; boundary=" + boundary, "User-Agent": USER_AGENT}
        attempt = 0
        while True:
            status, raw = self._send("POST", form["uploadUrl"], body, headers, timeout=600)
            if 200 <= status < 300:
                return form["fileUrl"]
            if (status >= 500 or status == 429) and attempt < self.retries:
                attempt += 1
                self.sleep(2 ** attempt)
                continue
            raise ApiError("Upload to storage failed (HTTP %d): %s" % (status, raw[:300].decode("utf-8", "replace")), status)

    def start(self, params):
        # Never retried: there is no idempotency key, and a second POST is a second billed job.
        try:
            created = self.call("POST", "/transcribe", params, retry=False)
        except ApiError as error:
            if error.status is None:
                raise ApiError(str(error) + "\nThe job may or may not have started. Do not start it again"
                               " without checking: a second request is a second billed job.")
            raise
        if not created.get("jobId"):
            raise ApiError("KolWrite did not return a job id.")
        return created["jobId"]

    def wait(self, job_id, timeout):
        """Poll with backoff (3 s growing to 15 s) until COMPLETED or FAILED or timeout seconds of waiting."""
        waited, delay = 0.0, 3.0
        while True:
            job = self.call("GET", "/transcribe/" + urllib.parse.quote(job_id, safe=""))
            if job.get("status") in ("COMPLETED", "FAILED") or waited + delay > timeout:
                return job
            self.sleep(delay)
            waited += delay
            delay = min(delay * 1.5, 15.0)


def api_error(status, raw, redact):
    text = raw.decode("utf-8", "replace")
    try:
        body = json.loads(text)
        detail = body.get("error") or body.get("message") or text
    except (ValueError, AttributeError):
        detail = text.strip()[:300]
    detail = redact(detail)
    retryable = status == 429 or status >= 500
    if status == 403 and str(detail).lower().startswith("insufficient credits"):
        hint = " Add credits at %s/billing." % CONSOLE
    elif status == 403 and detail in ("API key is required", "Invalid or inactive API key"):
        hint = " Check KOLWRITE_API_KEY (keys are on the API Keys page of %s)." % CONSOLE
    else:
        hint = ""
    return ApiError("HTTP %d: %s.%s" % (status, str(detail).rstrip("."), hint), status, retryable)


def multipart(fields, filename, content_type, data):
    boundary = "kolwrite-" + uuid.uuid4().hex
    parts = []
    for name, value in fields.items():
        parts.append(("--%s\r\nContent-Disposition: form-data; name=\"%s\"\r\n\r\n%s\r\n" % (boundary, name, value)).encode("utf-8"))
    parts.append(("--%s\r\nContent-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\nContent-Type: %s\r\n\r\n"
                  % (boundary, filename.replace('"', ""), content_type)).encode("utf-8"))
    parts.append(data)
    parts.append(("\r\n--%s--\r\n" % boundary).encode("utf-8"))
    return b"".join(parts), boundary


def default_model(args):
    if args.model:
        return args.model
    return "oracle_mini" if args.translate_to else "base"


def render(result, fmt):
    if isinstance(result, str):
        return result if result.endswith("\n") else result + "\n"
    if fmt == "json":
        return json.dumps(result, ensure_ascii=False, indent=2) + "\n"
    return json.dumps(result, ensure_ascii=False) + "\n"


def emit(args, job, output_path=None):
    if args.json:
        summary = {key: job.get(key) for key in ("jobId", "status", "creditsUsed", "creditsBalance", "error") if key in job}
        if output_path:
            summary["output"] = output_path
        elif job.get("status") == "COMPLETED":
            summary["result"] = job.get("result")
        sys.stdout.write(json.dumps(summary, ensure_ascii=False) + "\n")


def finish(args, job, fmt):
    status = job.get("status")
    job_id = job.get("jobId", "")
    if status == "FAILED":
        emit(args, job)
        say("Job %s failed: %s" % (job_id, job.get("error") or "no reason given"))
        return EXIT_FAILED
    if status != "COMPLETED":
        emit(args, job)
        say("Job %s is still %s. It keeps running; check it later with: kolwrite.py job %s" % (job_id, status, job_id))
        return EXIT_TIMEOUT
    text = render(job.get("result"), fmt)
    if args.output:
        with open(args.output, "w", encoding="utf-8") as handle:
            handle.write(text)
    elif not args.json:
        sys.stdout.write(text)
    emit(args, job, args.output)
    cost = []
    if job.get("creditsUsed") is not None:
        cost.append("credits used %s (US$%.2f)" % (job["creditsUsed"], float(job["creditsUsed"]) * CREDIT_USD))
    if job.get("creditsBalance") is not None:
        cost.append("balance %s" % job["creditsBalance"])
    say("Done: job %s%s%s" % (job_id, "; " if cost else "", ", ".join(cost)) + (" -> " + args.output if args.output else ""))
    return EXIT_OK


def client_from_env(sleep=time.sleep):
    key = os.environ.get("KOLWRITE_API_KEY", "").strip()
    if not key:
        say("KOLWRITE_API_KEY is not set. A person creates a key on the API Keys page of %s\n"
            "(sign up, add credits, create a key), then: export KOLWRITE_API_KEY=<the key>" % CONSOLE)
        return None
    base = os.environ.get("KOLWRITE_API_BASE", API_BASE)
    host = urllib.parse.urlparse(base).hostname or ""
    if not base.startswith("https://") and host not in ("127.0.0.1", "localhost", "::1"):
        say("KOLWRITE_API_BASE must be an https URL.")
        return None
    return Client(key, base, sleep=sleep)


def cmd_transcribe(args, client):
    source = args.source
    if source.startswith(("http://", "https://")):
        audio_url = source
    elif os.path.isfile(source):
        say("Uploading %s" % os.path.basename(source))
        audio_url = client.upload(source)
    else:
        say("Not a file or an http(s) URL: %s" % source)
        return EXIT_USAGE
    params = {"audio_url": audio_url, "model": default_model(args), "output_format": args.format,
              "language": args.language or "auto"}
    if args.translate_to:
        params["translate_to"] = args.translate_to
    if args.speakers:
        params["include_diarization"] = True
    if args.num_speakers:
        params["num_speakers"] = args.num_speakers
    # SRT: one cue per segment unless word cues are asked for; JSON: word timings on.
    params["word_timestamps"] = bool(args.word_timestamps) if args.format == "srt" else args.format == "json"
    job_id = client.start(params)
    say("Started job %s (model %s)" % (job_id, params["model"]))
    if args.no_wait:
        emit(args, {"jobId": job_id, "status": "PENDING"})
        if not args.json:
            sys.stdout.write(job_id + "\n")
        return EXIT_OK
    job = client.wait(job_id, args.timeout)
    job.setdefault("jobId", job_id)
    return finish(args, job, args.format)


def cmd_job(args, client):
    job = client.call("GET", "/transcribe/" + urllib.parse.quote(args.job_id, safe=""))
    job.setdefault("jobId", args.job_id)
    payload = job.get("payload") if isinstance(job.get("payload"), dict) else {}
    return finish(args, job, payload.get("output_format", "json"))


def cmd_models(args):
    rows = [{"id": m[0], "name": m[1], "creditsPerAudioHour": m[2], "usdPerAudioHour": round(m[2] * CREDIT_USD, 2),
             "translate": m[3], "current": m[4]} for m in MODELS]
    if args.json:
        sys.stdout.write(json.dumps({"models": rows, "creditUsd": CREDIT_USD,
                                     "diarizationCreditsPerHour": DIARIZATION_CREDITS,
                                     "proDiarizationCreditsPerHour": PRO_DIARIZATION_CREDITS}) + "\n")
        return EXIT_OK
    sys.stdout.write("%-12s %-14s %8s %9s  %s\n" % ("model", "name", "credits/h", "US$/h", "translate"))
    for row in rows:
        sys.stdout.write("%-12s %-14s %8.2f %9.2f  %s\n" % (row["id"], row["name"], row["creditsPerAudioHour"],
                                                            row["usdPerAudioHour"], "yes" if row["translate"] else "no"))
    sys.stdout.write("Speakers (--speakers) add %.2f credits per audio hour. Metered per second, no minimum.\n" % DIARIZATION_CREDITS)
    return EXIT_OK


def cmd_languages(args):
    rows = [{"code": code, "name": name} for code, name in LANGUAGES]
    if args.json:
        sys.stdout.write(json.dumps({"languages": rows}, ensure_ascii=False) + "\n")
    else:
        for row in rows:
            sys.stdout.write("%-4s %s\n" % (row["code"], row["name"]))
        sys.stdout.write("Use the code with --language or --translate-to. Full table: %s\n" % LANGUAGES_URL)
    return EXIT_OK


def parser():
    top = argparse.ArgumentParser(prog="kolwrite.py", description="Transcribe audio and video with the KolWrite API. Key: KOLWRITE_API_KEY.")
    top.add_argument("--version", action="version", version="kolwrite.py " + VERSION)
    sub = top.add_subparsers(dest="command")
    model_ids = [m[0] for m in MODELS]
    codes = [code for code, _ in LANGUAGES]

    t = sub.add_parser("transcribe", help="transcribe a local file or a public media URL")
    t.add_argument("source", help="local audio or video file (up to 100 MiB) or an http(s) URL of a media file")
    t.add_argument("--model", choices=model_ids, help="default: base, or oracle_mini with --translate-to")
    t.add_argument("--language", choices=["auto"] + codes, help="spoken language code; default auto")
    t.add_argument("--format", choices=["txt", "srt", "json"], default="txt", help="default txt")
    t.add_argument("--speakers", action="store_true", help="label speakers (adds %.2f credits per hour)" % DIARIZATION_CREDITS)
    t.add_argument("--num-speakers", type=int, help="expected number of speakers, with --speakers")
    t.add_argument("--translate-to", choices=codes, help="translate into this language (Oracle models)")
    t.add_argument("--word-timestamps", action="store_true", help="SRT only: one cue per word instead of per segment")
    t.add_argument("-o", "--output", help="write the result to this file instead of stdout")
    t.add_argument("--timeout", type=float, default=3600, help="seconds to wait for the result (default 3600)")
    t.add_argument("--no-wait", action="store_true", help="print the job id and exit")
    t.add_argument("--json", action="store_true", help="machine output: one JSON object on stdout")

    j = sub.add_parser("job", help="status or result of a job")
    j.add_argument("job_id")
    j.add_argument("-o", "--output")
    j.add_argument("--json", action="store_true")

    m = sub.add_parser("models", help="models and prices")
    m.add_argument("--json", action="store_true")
    lang = sub.add_parser("languages", help="language codes")
    lang.add_argument("--json", action="store_true")
    return top


def main(argv=None, sleep=time.sleep):
    args = parser().parse_args(argv)
    if args.command == "models":
        return cmd_models(args)
    if args.command == "languages":
        return cmd_languages(args)
    if args.command not in ("transcribe", "job"):
        parser().print_help(sys.stderr)
        return EXIT_USAGE
    if args.command == "transcribe" and args.num_speakers is not None and args.num_speakers < 1:
        say("--num-speakers must be at least 1")
        return EXIT_USAGE
    client = client_from_env(sleep=sleep)
    if client is None:
        return EXIT_USAGE
    try:
        return cmd_transcribe(args, client) if args.command == "transcribe" else cmd_job(args, client)
    except ApiError as error:
        say("KolWrite: %s" % error)
        return EXIT_ERROR
    except KeyboardInterrupt:
        say("Stopped. A job that already started keeps running.")
        return EXIT_ERROR


LANGUAGES = [
    # code, English name. Generated from sites/kolwrite-console/scripts/seo/content/languages.json.
    ("af", "Afrikaans"), ("sq", "Albanian"), ("am", "Amharic"), ("ar", "Arabic"), ("hy", "Armenian"),
    ("as", "Assamese"), ("az", "Azerbaijani"), ("ba", "Bashkir"), ("eu", "Basque"), ("be", "Belarusian"),
    ("bn", "Bengali"), ("bs", "Bosnian"), ("br", "Breton"), ("bg", "Bulgarian"), ("my", "Burmese"), ("ca", "Catalan"),
    ("zh", "Chinese"), ("hr", "Croatian"), ("cs", "Czech"), ("da", "Danish"), ("nl", "Dutch"), ("en", "English"),
    ("et", "Estonian"), ("fo", "Faroese"), ("fi", "Finnish"), ("fr", "French"), ("gl", "Galician"),
    ("ka", "Georgian"), ("de", "German"), ("el", "Greek"), ("gu", "Gujarati"), ("ht", "Haitian Creole"),
    ("ha", "Hausa"), ("haw", "Hawaiian"), ("he", "Hebrew"), ("hi", "Hindi"), ("hu", "Hungarian"), ("is", "Icelandic"),
    ("id", "Indonesian"), ("it", "Italian"), ("ja", "Japanese"), ("jw", "Javanese"), ("kn", "Kannada"),
    ("kk", "Kazakh"), ("km", "Khmer"), ("ko", "Korean"), ("lo", "Lao"), ("la", "Latin"), ("lv", "Latvian"),
    ("ln", "Lingala"), ("lt", "Lithuanian"), ("lb", "Luxembourgish"), ("mk", "Macedonian"), ("mg", "Malagasy"),
    ("ms", "Malay"), ("ml", "Malayalam"), ("mt", "Maltese"), ("mi", "Maori"), ("mr", "Marathi"), ("mn", "Mongolian"),
    ("ne", "Nepali"), ("no", "Norwegian"), ("nn", "Nynorsk"), ("oc", "Occitan"), ("ps", "Pashto"), ("fa", "Persian"),
    ("pl", "Polish"), ("pt", "Portuguese"), ("pa", "Punjabi"), ("ro", "Romanian"), ("ru", "Russian"),
    ("sa", "Sanskrit"), ("sr", "Serbian"), ("sn", "Shona"), ("sd", "Sindhi"), ("si", "Sinhala"), ("sk", "Slovak"),
    ("sl", "Slovenian"), ("so", "Somali"), ("es", "Spanish"), ("su", "Sundanese"), ("sw", "Swahili"),
    ("sv", "Swedish"), ("tl", "Tagalog"), ("tg", "Tajik"), ("ta", "Tamil"), ("tt", "Tatar"), ("te", "Telugu"),
    ("th", "Thai"), ("bo", "Tibetan"), ("tr", "Turkish"), ("tk", "Turkmen"), ("uk", "Ukrainian"), ("ur", "Urdu"),
    ("uz", "Uzbek"), ("vi", "Vietnamese"), ("cy", "Welsh"), ("yi", "Yiddish"), ("yo", "Yoruba"), ("zu", "Zulu"),
]

if __name__ == "__main__":
    sys.exit(main())
