diff --git a/.gitignore b/.gitignore index c0443ad..e547d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ /google-cloud-functions/venv /google-cloud-functions/requirements.txt /google-cloud-functions/gcf-executor.zip + +/docker/requirements.txt + +**/**.pyc diff --git a/Makefile b/Makefile index 27b80f5..1b8628d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lambda gcf +.PHONY: lambda gcf docker docker-ip VERSION=2.0.3 CURRENT_DIR=$(shell pwd) @@ -19,3 +19,11 @@ gcf: mv $(CURRENT_DIR)/google-cloud-functions/main.bak $(CURRENT_DIR)/google-cloud-functions/main.py rm $(CURRENT_DIR)/google-cloud-functions/requirements.txt +docker: + cat requirements.txt $(CURRENT_DIR)/docker/extra_requirements.txt > $(CURRENT_DIR)/docker/requirements.txt + (cd docker && docker compose up --force-recreate --build -d) + rm $(CURRENT_DIR)/docker/requirements.txt + +docker-ip: + $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) + @echo "Your local executor's url is http://"$(IP)":8000" diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..76a45f8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9-slim + +ENV MAX_EXECUTABLE=8192 +ENV MAX_DATA_SIZE=512 + +WORKDIR /executor + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir --upgrade -r requirements.txt + +COPY executor . + +EXPOSE 8000 + +RUN adduser --system --no-create-home nonroot +USER nonroot + +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "16", "--timeout", "10"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..9a75fa0 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,11 @@ +version: "3.9" +services: + executor: + build: + context: . + restart: always + ports: + - "8000:8000" + # FOR TESTING ONLY + # volumes: + # - ./executor:/executor diff --git a/docker/executor/app.py b/docker/executor/app.py new file mode 100644 index 0000000..675f03e --- /dev/null +++ b/docker/executor/app.py @@ -0,0 +1,80 @@ +from flask import jsonify, Flask, request +import os +import shlex +import base64 +import sys +import io + +# preload +import preload + +runtime_version = "docker-executor:0.2.4" +app = Flask(__name__) + + +def get_env(env, flag): + if flag not in env: + raise Exception(flag + " is missing") + return int(env[flag]) + + +def bad_request(err): + return jsonify({"error": err}), 400 + + +@app.post("/") +def execute(): + base_env = os.environ.copy() + + MAX_EXECUTABLE = get_env(base_env, "MAX_EXECUTABLE") + MAX_DATA_SIZE = get_env(base_env, "MAX_DATA_SIZE") + + request_json = request.get_json(force=True) + + if "executable" not in request_json: + raise bad_request("Missing executable value") + executable = base64.b64decode(request_json["executable"]) + if len(executable) > MAX_EXECUTABLE: + raise bad_request("Executable exceeds max size") + if "calldata" not in request_json: + raise bad_request("Missing calldata value") + if len(request_json["calldata"]) > MAX_DATA_SIZE: + raise bad_request("Calldata exceeds max size") + + res = { + "returncode": 0, + "stdout": "", + "stderr": "", + "err": "", + "version": runtime_version, + } + + out = io.StringIO() + err = io.StringIO() + + try: + sys.stdout = out + sys.stderr = err + sys.argv = shlex.split("file " + request_json["calldata"]) + + env = os.environ.copy() + for key, value in request_json.get("env", {}).items(): + env[key] = value + os.environ.update(env) + + exec(executable.decode(), {"__name__": "__main__"}) + except BaseException as e: + res["returncode"] = 126 + res["err"] = "Execution fail" + finally: + os.environ.update(base_env) + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + res["stdout"] = out.getvalue()[:MAX_DATA_SIZE] + res["stderr"] = err.getvalue()[:MAX_DATA_SIZE] + + if len(res["stderr"]) != 0: + res["returncode"] = 1 + + return res diff --git a/docker/executor/preload.py b/docker/executor/preload.py new file mode 100644 index 0000000..023e030 --- /dev/null +++ b/docker/executor/preload.py @@ -0,0 +1,20 @@ +import aiodns +import aiohttp +import aiosignal +import async_timeout +import attrs +import bech32 +import ccxt +import certifi +import cffi +import charset_normalizer +import cryptography +import frozenlist +import idna +import multidict +import pycares +import pycparser +import requests +import urllib3 +import websocket +import yarl diff --git a/docker/extra_requirements.txt b/docker/extra_requirements.txt new file mode 100644 index 0000000..70c13bf --- /dev/null +++ b/docker/extra_requirements.txt @@ -0,0 +1,2 @@ +Flask==2.2.2 +gunicorn==20.1.0 diff --git a/scripts/data_source.py b/scripts/data_source.py new file mode 100644 index 0000000..55e0179 --- /dev/null +++ b/scripts/data_source.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import sys +import ccxt +import aiohttp +import asyncio +import os +import time + +if __name__ == "__main__": + try: + time.sleep(5) + print(sys.argv[1:]) + print(os.getpid()) + print(globals()) + except Exception as e: + print(str(e), file=sys.stderr) + sys.exit(1) diff --git a/scripts/load_test.js b/scripts/load_test.js new file mode 100644 index 0000000..879d401 --- /dev/null +++ b/scripts/load_test.js @@ -0,0 +1,33 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter } from "k6/metrics"; +import { b64encode } from "k6/encoding"; +let ErrorCount = new Counter("errors"); +const data = b64encode(open('./data_source.py')) + +export default function () { + let res = http.post("http://localhost:8000", JSON.stringify({ + "calldata": "TEST_ARG", + "env": { + "BAND_CHAIN_ID": "test-chain-id", + "BAND_EXTERNAL_ID": "test-external-id", + "BAND_REPORTER": "test-reporter", + "BAND_REQUEST_ID": "test-request-id", + "BAND_SIGNATURE": "test-signature", + "BAND_VALIDATOR": "test-validator" + }, + "executable": data, + "timeout": 10000 + }), { headers: { 'Content-Type': 'application/json' } }) + + console.log(res.body) + + let success = check(res, { + "status is 200": (r) => r.status === 200, + }); + + if (!success) { + ErrorCount.add(1); + } + +}