Docker executor - exec way

.gitignore

@@ -5,3 +5,7 @@
 /google-cloud-functions/venv
 /google-cloud-functions/requirements.txt
 /google-cloud-functions/gcf-executor.zip
+
+/docker/requirements.txt
+
+**/**.pyc

Makefile

@@ -1,4 +1,4 @@
-.PHONY: lambda gcf
+.PHONY: lambda gcf docker docker-ip
 
 VERSION=2.0.3
 CURRENT_DIR=$(shell pwd)
@@ -19,3 +19,11 @@ gcf:
 	mv $(CURRENT_DIR)/google-cloud-functions/main.bak $(CURRENT_DIR)/google-cloud-functions/main.py
 	rm $(CURRENT_DIR)/google-cloud-functions/requirements.txt
 
+docker:
+	cat requirements.txt $(CURRENT_DIR)/docker/extra_requirements.txt > $(CURRENT_DIR)/docker/requirements.txt
+	(cd docker && docker compose up --force-recreate --build -d)
+	rm $(CURRENT_DIR)/docker/requirements.txt
+
+docker-ip:
+	$(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .))
+	@echo "Your local executor's url is http://"$(IP)":8000"

docker/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.9-slim
+
+ENV MAX_EXECUTABLE=8192
+ENV MAX_DATA_SIZE=512
+
+WORKDIR /executor
+
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY executor .
+
+EXPOSE 8000
+
+RUN adduser --system --no-create-home nonroot
+USER nonroot
+
+CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "16", "--timeout", "10"]

docker/docker-compose.yaml

@@ -0,0 +1,11 @@
+version: "3.9"
+services:
+  executor:
+    build:
+      context: .
+    restart: always
+    ports:
+      - "8000:8000"
+    # FOR TESTING ONLY
+    # volumes:
+    #   - ./executor:/executor

docker/executor/app.py

@@ -0,0 +1,80 @@
+from flask import jsonify, Flask, request
+import os
+import shlex
+import base64
+import sys
+import io
+
+# preload
+import preload
+
+runtime_version = "docker-executor:0.2.4"
+app = Flask(__name__)
+
+
+def get_env(env, flag):
+    if flag not in env:
+        raise Exception(flag + " is missing")
+    return int(env[flag])
+
+
+def bad_request(err):
+    return jsonify({"error": err}), 400
+
+
+@app.post("/")
+def execute():
+    base_env = os.environ.copy()
+
+    MAX_EXECUTABLE = get_env(base_env, "MAX_EXECUTABLE")
+    MAX_DATA_SIZE = get_env(base_env, "MAX_DATA_SIZE")
+
+    request_json = request.get_json(force=True)
+
+    if "executable" not in request_json:
+        raise bad_request("Missing executable value")
+    executable = base64.b64decode(request_json["executable"])
+    if len(executable) > MAX_EXECUTABLE:
+        raise bad_request("Executable exceeds max size")
+    if "calldata" not in request_json:
+        raise bad_request("Missing calldata value")
+    if len(request_json["calldata"]) > MAX_DATA_SIZE:
+        raise bad_request("Calldata exceeds max size")
+
+    res = {
+        "returncode": 0,
+        "stdout": "",
+        "stderr": "",
+        "err": "",
+        "version": runtime_version,
+    }
+
+    out = io.StringIO()
+    err = io.StringIO()
+
+    try:
+        sys.stdout = out
+        sys.stderr = err
+        sys.argv = shlex.split("file " + request_json["calldata"])
+
+        env = os.environ.copy()
+        for key, value in request_json.get("env", {}).items():
+            env[key] = value
+        os.environ.update(env)
+
+        exec(executable.decode(), {"__name__": "__main__"})
+    except BaseException as e:
+        res["returncode"] = 126
+        res["err"] = "Execution fail"
+    finally:
+        os.environ.update(base_env)
+        sys.stdout = sys.__stdout__
+        sys.stderr = sys.__stderr__
+
+        res["stdout"] = out.getvalue()[:MAX_DATA_SIZE]
+        res["stderr"] = err.getvalue()[:MAX_DATA_SIZE]
+
+        if len(res["stderr"]) != 0:
+            res["returncode"] = 1
+
+        return res

docker/executor/preload.py

@@ -0,0 +1,20 @@
+import aiodns
+import aiohttp
+import aiosignal
+import async_timeout
+import attrs
+import bech32
+import ccxt
+import certifi
+import cffi
+import charset_normalizer
+import cryptography
+import frozenlist
+import idna
+import multidict
+import pycares
+import pycparser
+import requests
+import urllib3
+import websocket
+import yarl

docker/extra_requirements.txt

@@ -0,0 +1,2 @@
+Flask==2.2.2
+gunicorn==20.1.0

scripts/data_source.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import sys
+import ccxt
+import aiohttp
+import asyncio
+import os
+import time
+
+if __name__ == "__main__":
+    try:
+        time.sleep(5)
+        print(sys.argv[1:])
+        print(os.getpid())
+        print(globals())
+    except Exception as e:
+        print(str(e), file=sys.stderr)
+        sys.exit(1)

scripts/load_test.js

@@ -0,0 +1,33 @@
+import http from "k6/http";
+import { check, sleep } from "k6";
+import { Counter } from "k6/metrics";
+import { b64encode } from "k6/encoding";
+let ErrorCount = new Counter("errors");
+const data = b64encode(open('./data_source.py'))
+
+export default function () {
+  let res = http.post("http://localhost:8000", JSON.stringify({
+    "calldata": "TEST_ARG",
+    "env": {
+      "BAND_CHAIN_ID": "test-chain-id",
+      "BAND_EXTERNAL_ID": "test-external-id",
+      "BAND_REPORTER": "test-reporter",
+      "BAND_REQUEST_ID": "test-request-id",
+      "BAND_SIGNATURE": "test-signature",
+      "BAND_VALIDATOR": "test-validator"
+    },
+    "executable": data,
+    "timeout": 10000
+  }), { headers: { 'Content-Type': 'application/json' } })
+
+  console.log(res.body)
+
+  let success = check(res, {
+    "status is 200": (r) => r.status === 200,
+  });
+
+  if (!success) {
+    ErrorCount.add(1);
+  }
+
+}