From 2ae699b0c60e2e6fdb0651c8df19dd5b345034cf Mon Sep 17 00:00:00 2001
From: Nathan Sarrazin <sarrazin.nathan@gmail.com>
Date: Sun, 19 Mar 2023 12:09:15 +0100
Subject: [PATCH] basic working version

---
 .gitignore                        |  5 +++
 Dockerfile.api                    | 27 +++++++++++++
 Dockerfile.web                    |  0
 api/main.py                       | 67 +++++++++++++++++++++++++++++++
 api/requirements.txt              |  2 +
 docker-compose.yml                | 12 ++++++
 weights/put_your_weights_here.txt |  0
 7 files changed, 113 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Dockerfile.api
 create mode 100644 Dockerfile.web
 create mode 100644 api/main.py
 create mode 100644 api/requirements.txt
 create mode 100644 docker-compose.yml
 create mode 100644 weights/put_your_weights_here.txt

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..017ff1d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+api/__pycache__
+api/magic.dat
+
+weights/**
+!weights/put_your_weights_here.txt
diff --git a/Dockerfile.api b/Dockerfile.api
new file mode 100644
index 0000000..d1781d6
--- /dev/null
+++ b/Dockerfile.api
@@ -0,0 +1,27 @@
+FROM ubuntu:latest as builder
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    zlib1g-dev \
+    libbz2-dev \
+    liblzma-dev \
+    autoconf \
+    git \
+    wget
+
+WORKDIR /tmp
+
+RUN git clone https://github.com/ggerganov/llama.cpp.git -b mmap
+RUN cd llama.cpp && sed -i 's/{ 5120, 2 },/{ 5120, 1 },/g' main.cpp && make
+
+FROM ubuntu:latest
+WORKDIR /usr/src/app
+
+RUN apt update
+RUN apt-get install -y python3-pip
+RUN pip install --upgrade pip
+
+COPY ./api/requirements.txt requirements.txt
+COPY --from=builder /tmp/llama.cpp/main /usr/bin/llama
+
+RUN pip install -r requirements.txt
diff --git a/Dockerfile.web b/Dockerfile.web
new file mode 100644
index 0000000..e69de29
diff --git a/api/main.py b/api/main.py
new file mode 100644
index 0000000..186943d
--- /dev/null
+++ b/api/main.py
@@ -0,0 +1,67 @@
+from typing import Union
+from fastapi import FastAPI
+import subprocess, os
+
+app = FastAPI()
+
+
+@app.get("/generate")
+def generate(
+    model: str = "ggml-alpaca-13b-q4.bin",
+    prompt: str = "What is the first letter of the alphabet?",
+    temp: float = 0.8,
+    top_k: int = 40,
+    top_p: float = 0.9,
+    repeast_last_n: int = 64,
+    repeat_penalty: float = 1.3,
+    stop_token: str = None,
+):
+
+    prompter = (
+        lambda prompt: f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
+
+### Instruction:
+{prompt}
+
+### Response:
+"""
+    )
+
+    args = (
+        "llama",
+        "--model",
+        "weights/" + model,
+        "--prompt",
+        prompter(prompt),
+        "--temp",
+        str(temp),
+        "--top_k",
+        str(top_k),
+        "--top_p",
+        str(top_p),
+    )
+
+    print(args)
+
+    procLlama = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    while True:
+        if procLlama.poll() is not None:
+            break
+
+    output = procLlama.stdout.read().decode("utf-8")
+
+    splits = output.split("###")
+
+    return {
+        "input": splits[1].lstrip(" Instruction:\n").rstrip("\n\n"),
+        "output": splits[2].lstrip(" Response:\n"),
+    }
+
+
+@app.get("/models")
+def models():
+    files = os.listdir("weights")
+    files.remove("put_your_weights_here.txt")
+
+    return files
diff --git a/api/requirements.txt b/api/requirements.txt
new file mode 100644
index 0000000..f0615cf
--- /dev/null
+++ b/api/requirements.txt
@@ -0,0 +1,2 @@
+fastapi
+uvicorn
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..d9bbed9
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,12 @@
+version: "3.9"
+services:
+  api:
+    build: 
+      context: .
+      dockerfile: Dockerfile.api
+    command: uvicorn main:app --reload --host 0.0.0.0 --port 9123
+    volumes:
+      - ./api:/usr/src/app/
+      - ./weights:/usr/src/app/weights
+    ports:
+      - 9123:9123
\ No newline at end of file
diff --git a/weights/put_your_weights_here.txt b/weights/put_your_weights_here.txt
new file mode 100644
index 0000000..e69de29