From 1688113be0db0bf0fde654434342c4a89a905360 Mon Sep 17 00:00:00 2001 From: colmazia Date: Thu, 5 Jan 2023 13:53:07 +0700 Subject: [PATCH 01/13] add docker executor --- docker/Dockerfile | 13 ++++ docker/docker-executor/app.py | 98 +++++++++++++++++++++++++ docker/docker-executor/requirements.txt | 31 ++++++++ 3 files changed, 142 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/docker-executor/app.py create mode 100644 docker/docker-executor/requirements.txt diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8a8e5f0 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim-buster + +WORKDIR /executor +COPY docker-executor /executor + +ENV MAX_EXECUTABLE=8192 +ENV MAX_DATA_SIZE=512 + +EXPOSE 5000 + +RUN pip3 install -r requirements.txt + +CMD [ "gunicorn", "app:app", "-b", "0.0.0.0:5000"] diff --git a/docker/docker-executor/app.py b/docker/docker-executor/app.py new file mode 100644 index 0000000..9ba6f25 --- /dev/null +++ b/docker/docker-executor/app.py @@ -0,0 +1,98 @@ +from flask import jsonify, Flask, request +import os +import shlex +import subprocess +import base64 + +# Set environment flag of MAX_EXECUTABLE, MAX_DATA_SIZE + + +runtime_version = "google-cloud-function:2.0.3" +app = Flask(__name__) + +def get_env(env, flag): + if flag not in env: + raise Exception(flag + " is missing") + return int(env[flag]) + + +def success(returncode, stdout, stderr, err): + return ( + jsonify( + { + "returncode": returncode, + "stdout": stdout, + "stderr": stderr, + "err": err, + "version": runtime_version, + } + ), + 200, + ) + + +def bad_request(err): + return jsonify({"error": err}), 400 + +@app.route('/', methods=['POST']) +def execute(): + """Responds to any HTTP request. + Args: + request (flask.Request): HTTP request object. + Returns: + The response text or any set of values that can be turned into a + Response object using + `make_response `. + """ + env = os.environ.copy() + + MAX_EXECUTABLE = get_env(env, "MAX_EXECUTABLE") + MAX_DATA_SIZE = get_env(env, "MAX_DATA_SIZE") + + request_json = request.get_json(force=True) + if "executable" not in request_json: + return bad_request("Missing executable value") + executable = base64.b64decode(request_json["executable"]) + if len(executable) > MAX_EXECUTABLE: + return bad_request("Executable exceeds max size") + if "calldata" not in request_json: + return bad_request("Missing calldata value") + if len(request_json["calldata"]) > MAX_DATA_SIZE: + return bad_request("Calldata exceeds max size") + if "timeout" not in request_json: + return bad_request("Missing timeout value") + try: + timeout = int(request_json["timeout"]) + except ValueError: + return bad_request("Timeout format invalid") + + path = "/tmp/execute.sh" + with open(path, "w") as f: + f.write(executable.decode()) + + os.chmod(path, 0o775) + try: + env = os.environ.copy() + for key, value in request_json.get("env", {}).items(): + env[key] = value + + proc = subprocess.Popen( + [path] + shlex.split(request_json["calldata"]), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + proc.wait(timeout=(timeout / 1000)) + returncode = proc.returncode + stdout = proc.stdout.read(MAX_DATA_SIZE).decode() + stderr = proc.stderr.read(MAX_DATA_SIZE).decode() + return success(returncode, stdout, stderr, "") + except OSError: + return success(126, "", "", "Execution fail") + except subprocess.TimeoutExpired: + return success(111, "", "", "Execution time limit exceeded") + +@app.route('/', methods=['GET']) +def hello_world(): + return 'Hello there' diff --git a/docker/docker-executor/requirements.txt b/docker/docker-executor/requirements.txt new file mode 100644 index 0000000..b2f78ca --- /dev/null +++ b/docker/docker-executor/requirements.txt @@ -0,0 +1,31 @@ +aiodns==3.0.0 +aiohttp==3.8.1 +aiosignal==1.2.0 +async-timeout==4.0.2 +asynctest==0.13.0 +attrs==21.4.0 +bech32==1.2.0 +ccxt==1.87.48 +certifi==2022.5.18.1 +cffi==1.15.0 +charset-normalizer==2.0.12 +click==8.1.3 +cryptography==37.0.2 +Flask==2.1.2 +frozenlist==1.3.0 +gunicorn==20.1.0 +idna==3.3 +importlib-metadata==4.11.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +multidict==6.0.2 +pycares==4.1.2 +pycparser==2.21 +requests==2.28.0 +typing-extensions==4.4.0 +urllib3==1.26.9 +websocket-client==1.3.2 +Werkzeug==2.1.2 +yarl==1.7.2 +zipp==3.8.0 From 7b0014e0fa372689a52a7672be528b166316919e Mon Sep 17 00:00:00 2001 From: colmazia Date: Thu, 5 Jan 2023 13:54:32 +0700 Subject: [PATCH 02/13] remove get endpoint --- docker/docker-executor/app.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/docker-executor/app.py b/docker/docker-executor/app.py index 9ba6f25..c335061 100644 --- a/docker/docker-executor/app.py +++ b/docker/docker-executor/app.py @@ -92,7 +92,3 @@ def execute(): return success(126, "", "", "Execution fail") except subprocess.TimeoutExpired: return success(111, "", "", "Execution time limit exceeded") - -@app.route('/', methods=['GET']) -def hello_world(): - return 'Hello there' From 2fafed6490c27ee9fbf9915a874d513d753d5e44 Mon Sep 17 00:00:00 2001 From: colmazia Date: Thu, 2 Feb 2023 13:42:34 +0700 Subject: [PATCH 03/13] fast api and gunicorn --- docker/Dockerfile | 5 +- .../__pycache__/app.cpython-37.pyc | Bin 0 -> 2874 bytes docker/docker-executor/app.py | 114 +++++++++--------- docker/docker-executor/requirements.txt | 9 +- 4 files changed, 68 insertions(+), 60 deletions(-) create mode 100644 docker/docker-executor/__pycache__/app.cpython-37.pyc diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a8e5f0..8fdb873 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-buster +FROM python:3.9-slim-buster WORKDIR /executor COPY docker-executor /executor @@ -9,5 +9,6 @@ ENV MAX_DATA_SIZE=512 EXPOSE 5000 RUN pip3 install -r requirements.txt +RUN chmod 750 . -CMD [ "gunicorn", "app:app", "-b", "0.0.0.0:5000"] +CMD [ "gunicorn", "app:app", "-b", "0.0.0.0:5000", "--worker-class", "uvicorn.workers.UvicornWorker", "--workers", "48", "--timeout", "30"] diff --git a/docker/docker-executor/__pycache__/app.cpython-37.pyc b/docker/docker-executor/__pycache__/app.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5100f269abc67928d5e01ed0ae7162124d25896a GIT binary patch literal 2874 zcmZt|U2ojRahD%Q9`8$*^=ZWp6SqOHg>-HkxPjmrm8~;sAW;-au8l*A3y-^ZM;(v6 zEca|l;Ps*Mt5HAnsp!M$vHu}2dFyZ3*S^$$AwYr7khJ8qUV<~bvopIhv$He%pjLAT zw9kI?z5ll=A^*a`+h#!Lckm}K0WiX7NXE27$67}l>m8jUO$&{VkzuoAX1LTTW!UOi z8MZq%(>rCZap#&#&lzztqGUSL&L8&Fn%rLhKEcu7F+ z8q*F*^V%EGgfw+!9Vc-V^iP%fDD;w%vIPHj$`iSFwlRp~LC81#FrKo_{xtGs5J%g; z+Pbs#YhOcgXoz+AAHtuw08*NgktLQS^cY=*9sf)v&{Db%eb`ysJFB=s;*NtP38Fz$ zQ|0!F&nLLJ(t}6>Md#6TW%NUD@DC!&u<8%{ttSZ=Nh^*9o{+6w{+x&LWXvPk>Uj}s zF;9A%Njw!k-xSkG24mi0u|MLXK%4x8`%@WJrTRyLnET)<=ixdreb0g3GCh{Frf7N+gS?GQ(U5unBhLvXZIq^CVePLguH`*OvWN)&4V(~+TAYBtxEus5!_$DhV|M0%%=U^u^r8Xp zP9Y{ff!&hqyr@IPI*fjxEoJO~d3e;M${KSyj9GH27TSXgJw&m)ptvN*bgo^H8IgJl zUOCrhH0x>5)1+~1GMyQ(=uBs3szYdR%MvSLC`)T@r$AjN7j$OGa%!B@l#UGXHE>wS z!C)RFLKv4bi8Aopco9+!_+A1Xt}^gdGQPjUxQgS`1?d211vzKtQxdHiL{>(Q_>RI+ zf&)vehJ4Qms{>zSov?*aXc%5tE*`Vot~E1b)ZHz?%uelj zJvHWy3$TEar-UtkMGjz9kXit)l?#oM)9cG7y-y^T-xA&hCdHr}PwFXo!&sTDbFCZ19NMFtVHMaW7oK@1w@p@{qwGG17U*&;u zgWV`XWFzkZ&j+(Az?<`1sdhn8+gT09AHFlLOLzWJZm0Y>t){iK&Nj}CL%6{5_Q-jy ziBjRYclg)`Yy-ZV1>g1KpJbeO-otq--;IOyY=Fm(U8LuxLv|s z$xoyki)>H0FK*m-pL-$XZC4IG>H1#e_J9j=5oa#sRM4)_BG(v}$TxDQn1qYDZ^qt; z!^JK(xi^$@vfXOs`c7g$j<;elz=WE#ez|oAGHKAdozFhXZ2FP>{cq3;RsUk=S-1VH z{qV`r&V$d}s)pm;oui%Z;p0vl@2J_2A#E!s&(K~kIBhUBpL0md;hF=Oz5^r>fk$`zhmD zWz$0*39OjP`2z-B8_R|=$0HU9WyBL6Df5L0B-Fq{!t{sZm?<5WRwhi06Xhh+-bBP$ z!IZh53njylpC~h#gn@(%{&>Hg<#T1c@B%4TVMl?5GtE0aWHP;NjS5< z(nlO#OxcwKBA9WdQ(m$Ew^$cP{&QF|=>Z^)qdC;ks_^KrX&oLOh;6!1U=4b8n0KM5 z;j?57I5p_spmoili_kaVv-Jg_IkW`a*sB+C5&Gg0?9yC>+l5y%E9V01adygC42a+2 z@G}H=5&QzdA^>G~yOB5M-LBH%wUh@3z9+Jf$j`(Sa8-D=$$Lw*eGg828uI%XuL&O8 TQehgHX@csSU9uatZX5pvF+SHX literal 0 HcmV?d00001 diff --git a/docker/docker-executor/app.py b/docker/docker-executor/app.py index c335061..db587df 100644 --- a/docker/docker-executor/app.py +++ b/docker/docker-executor/app.py @@ -1,4 +1,5 @@ -from flask import jsonify, Flask, request +from fastapi import FastAPI, Request, HTTPException +import shutil import os import shlex import subprocess @@ -6,89 +7,88 @@ # Set environment flag of MAX_EXECUTABLE, MAX_DATA_SIZE - -runtime_version = "google-cloud-function:2.0.3" -app = Flask(__name__) +runtime_version = "docker-executor:0.2.4" +app = FastAPI() def get_env(env, flag): if flag not in env: raise Exception(flag + " is missing") return int(env[flag]) - -def success(returncode, stdout, stderr, err): - return ( - jsonify( - { - "returncode": returncode, - "stdout": stdout, - "stderr": stderr, - "err": err, - "version": runtime_version, - } - ), - 200, - ) - - -def bad_request(err): - return jsonify({"error": err}), 400 - -@app.route('/', methods=['POST']) -def execute(): - """Responds to any HTTP request. - Args: - request (flask.Request): HTTP request object. - Returns: - The response text or any set of values that can be turned into a - Response object using - `make_response `. - """ +@app.post("/") +async def execute(request: Request): env = os.environ.copy() MAX_EXECUTABLE = get_env(env, "MAX_EXECUTABLE") MAX_DATA_SIZE = get_env(env, "MAX_DATA_SIZE") - request_json = request.get_json(force=True) + request_json = await request.json() + if "executable" not in request_json: - return bad_request("Missing executable value") + raise HTTPException(status_code=400, detail="Missing executable value") executable = base64.b64decode(request_json["executable"]) if len(executable) > MAX_EXECUTABLE: - return bad_request("Executable exceeds max size") + raise HTTPException(status_code=400, detail="Executable exceeds max size") if "calldata" not in request_json: - return bad_request("Missing calldata value") + raise HTTPException(status_code=400, detail="Missing calldata value") if len(request_json["calldata"]) > MAX_DATA_SIZE: - return bad_request("Calldata exceeds max size") + raise HTTPException(status_code=400, detail="Calldata exceeds max size") if "timeout" not in request_json: - return bad_request("Missing timeout value") + raise HTTPException(status_code=400, detail="Missing timeout value") try: timeout = int(request_json["timeout"]) except ValueError: - return bad_request("Timeout format invalid") + raise HTTPException(status_code=400, detail="Timeout format invalid") - path = "/tmp/execute.sh" - with open(path, "w") as f: + user_folder = request_json["env"]["BAND_REQUEST_ID"]+"-"+request_json["env"]["BAND_EXTERNAL_ID"] + os.mkdir(user_folder) + path = user_folder+"/execute.sh" + with open(path, "w+") as f: f.write(executable.decode()) - os.chmod(path, 0o775) + os.chmod(path, 0o777) try: env = os.environ.copy() for key, value in request_json.get("env", {}).items(): env[key] = value + os.environ.update(env) + # proc = subprocess.Popen( + # [path] + shlex.split(request_json["calldata"]), + # env=env, + # stdout=subprocess.PIPE, + # stderr=subprocess.PIPE, + # ) - proc = subprocess.Popen( - [path] + shlex.split(request_json["calldata"]), - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - proc.wait(timeout=(timeout / 1000)) - returncode = proc.returncode - stdout = proc.stdout.read(MAX_DATA_SIZE).decode() - stderr = proc.stderr.read(MAX_DATA_SIZE).decode() - return success(returncode, stdout, stderr, "") + # proc.wait(timeout=(timeout / 1000)) + # returncode = proc.returncode + # stdout = proc.stdout.read(MAX_DATA_SIZE).decode() + # stderr = proc.stderr.read(MAX_DATA_SIZE).decode() + returnCode = os.system(path +" "+ ' '.join(shlex.split(request_json["calldata"])) + " > output.txt 2> error.txt") + output = open("output.txt").read(MAX_DATA_SIZE) + error = open("error.txt").read(MAX_DATA_SIZE) + shutil.rmtree(user_folder) + return { + "returncode": returnCode, + "stdout": output, + "stderr": error, + "err": "", + "version": runtime_version, + } except OSError: - return success(126, "", "", "Execution fail") + shutil.rmtree(user_folder) + return { + "returncode": 126, + "stdout": "", + "stderr": "", + "err": "Execution fail", + "version": runtime_version, + } except subprocess.TimeoutExpired: - return success(111, "", "", "Execution time limit exceeded") + shutil.rmtree(user_folder) + return { + "returncode": 111, + "stdout": "", + "stderr": "", + "err": "Execution time limit exceeded", + "version": runtime_version, + } diff --git a/docker/docker-executor/requirements.txt b/docker/docker-executor/requirements.txt index b2f78ca..ce33848 100644 --- a/docker/docker-executor/requirements.txt +++ b/docker/docker-executor/requirements.txt @@ -1,19 +1,22 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 +anyio<5,>=3.4.0 async-timeout==4.0.2 asynctest==0.13.0 attrs==21.4.0 bech32==1.2.0 -ccxt==1.87.48 +ccxt==1.87.1 certifi==2022.5.18.1 cffi==1.15.0 charset-normalizer==2.0.12 click==8.1.3 cryptography==37.0.2 +fastapi==0.89.1 Flask==2.1.2 frozenlist==1.3.0 gunicorn==20.1.0 +h11==0.14.0 idna==3.3 importlib-metadata==4.11.4 itsdangerous==2.1.2 @@ -22,9 +25,13 @@ MarkupSafe==2.1.1 multidict==6.0.2 pycares==4.1.2 pycparser==2.21 +pydantic==1.10.4 requests==2.28.0 +sniffio>=1.1 +starlette==0.22.0 typing-extensions==4.4.0 urllib3==1.26.9 +uvicorn==0.20.0 websocket-client==1.3.2 Werkzeug==2.1.2 yarl==1.7.2 From 1712eed6cdaf757cf82a289794a026e07df35c05 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:27:26 +0700 Subject: [PATCH 04/13] optimize local executor --- .gitignore | 4 + Makefile | 8 +- docker/Dockerfile | 23 +++-- docker/docker-compose.yaml | 11 ++ .../__pycache__/app.cpython-37.pyc | Bin 2874 -> 0 bytes docker/docker-executor/app.py | 94 ------------------ docker/docker-executor/requirements.txt | 38 ------- docker/executor/app.py | 76 ++++++++++++++ docker/executor/preload.py | 20 ++++ docker/extra_requirements.txt | 3 + scripts/data_source.py | 17 ++++ scripts/load_test.js | 31 ++++++ 12 files changed, 184 insertions(+), 141 deletions(-) create mode 100644 docker/docker-compose.yaml delete mode 100644 docker/docker-executor/__pycache__/app.cpython-37.pyc delete mode 100644 docker/docker-executor/app.py delete mode 100644 docker/docker-executor/requirements.txt create mode 100644 docker/executor/app.py create mode 100644 docker/executor/preload.py create mode 100644 docker/extra_requirements.txt create mode 100644 scripts/data_source.py create mode 100644 scripts/load_test.js diff --git a/.gitignore b/.gitignore index c0443ad..e547d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ /google-cloud-functions/venv /google-cloud-functions/requirements.txt /google-cloud-functions/gcf-executor.zip + +/docker/requirements.txt + +**/**.pyc diff --git a/Makefile b/Makefile index 27b80f5..3e18317 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lambda gcf +.PHONY: lambda gcf docker VERSION=2.0.3 CURRENT_DIR=$(shell pwd) @@ -19,3 +19,9 @@ gcf: mv $(CURRENT_DIR)/google-cloud-functions/main.bak $(CURRENT_DIR)/google-cloud-functions/main.py rm $(CURRENT_DIR)/google-cloud-functions/requirements.txt +docker: + cat requirements.txt $(CURRENT_DIR)/docker/extra_requirements.txt > $(CURRENT_DIR)/docker/requirements.txt + (cd docker && docker compose up --force-recreate --build -d) + rm $(CURRENT_DIR)/docker/requirements.txt + $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) + @echo "Your local executor's url is "$(IP)":8000" diff --git a/docker/Dockerfile b/docker/Dockerfile index 8fdb873..1af62c5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,14 +1,21 @@ -FROM python:3.9-slim-buster - -WORKDIR /executor -COPY docker-executor /executor +FROM python:3.9-slim ENV MAX_EXECUTABLE=8192 ENV MAX_DATA_SIZE=512 -EXPOSE 5000 +WORKDIR /executor + +COPY requirements.txt requirements.txt +RUN pip install --no-cache-dir --upgrade -r requirements.txt + +COPY executor . + +EXPOSE 8000 + +RUN adduser --system --no-create-home nonroot +USER nonroot -RUN pip3 install -r requirements.txt -RUN chmod 750 . +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "-k", "uvicorn.workers.UvicornWorker", "--workers", "32", "--timeout", "0"] -CMD [ "gunicorn", "app:app", "-b", "0.0.0.0:5000", "--worker-class", "uvicorn.workers.UvicornWorker", "--workers", "48", "--timeout", "30"] +# FOR TESTING ONLY +# CMD ["uvicorn","app:app", "--reload", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..653af6f --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,11 @@ +version: "3.9" +services: + executor: + build: + context: . + restart: always + # FOR TESTING ONLY + # ports: + # - "8000:8000" + # volumes: + # - ./executor:/executor diff --git a/docker/docker-executor/__pycache__/app.cpython-37.pyc b/docker/docker-executor/__pycache__/app.cpython-37.pyc deleted file mode 100644 index 5100f269abc67928d5e01ed0ae7162124d25896a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2874 zcmZt|U2ojRahD%Q9`8$*^=ZWp6SqOHg>-HkxPjmrm8~;sAW;-au8l*A3y-^ZM;(v6 zEca|l;Ps*Mt5HAnsp!M$vHu}2dFyZ3*S^$$AwYr7khJ8qUV<~bvopIhv$He%pjLAT zw9kI?z5ll=A^*a`+h#!Lckm}K0WiX7NXE27$67}l>m8jUO$&{VkzuoAX1LTTW!UOi z8MZq%(>rCZap#&#&lzztqGUSL&L8&Fn%rLhKEcu7F+ z8q*F*^V%EGgfw+!9Vc-V^iP%fDD;w%vIPHj$`iSFwlRp~LC81#FrKo_{xtGs5J%g; z+Pbs#YhOcgXoz+AAHtuw08*NgktLQS^cY=*9sf)v&{Db%eb`ysJFB=s;*NtP38Fz$ zQ|0!F&nLLJ(t}6>Md#6TW%NUD@DC!&u<8%{ttSZ=Nh^*9o{+6w{+x&LWXvPk>Uj}s zF;9A%Njw!k-xSkG24mi0u|MLXK%4x8`%@WJrTRyLnET)<=ixdreb0g3GCh{Frf7N+gS?GQ(U5unBhLvXZIq^CVePLguH`*OvWN)&4V(~+TAYBtxEus5!_$DhV|M0%%=U^u^r8Xp zP9Y{ff!&hqyr@IPI*fjxEoJO~d3e;M${KSyj9GH27TSXgJw&m)ptvN*bgo^H8IgJl zUOCrhH0x>5)1+~1GMyQ(=uBs3szYdR%MvSLC`)T@r$AjN7j$OGa%!B@l#UGXHE>wS z!C)RFLKv4bi8Aopco9+!_+A1Xt}^gdGQPjUxQgS`1?d211vzKtQxdHiL{>(Q_>RI+ zf&)vehJ4Qms{>zSov?*aXc%5tE*`Vot~E1b)ZHz?%uelj zJvHWy3$TEar-UtkMGjz9kXit)l?#oM)9cG7y-y^T-xA&hCdHr}PwFXo!&sTDbFCZ19NMFtVHMaW7oK@1w@p@{qwGG17U*&;u zgWV`XWFzkZ&j+(Az?<`1sdhn8+gT09AHFlLOLzWJZm0Y>t){iK&Nj}CL%6{5_Q-jy ziBjRYclg)`Yy-ZV1>g1KpJbeO-otq--;IOyY=Fm(U8LuxLv|s z$xoyki)>H0FK*m-pL-$XZC4IG>H1#e_J9j=5oa#sRM4)_BG(v}$TxDQn1qYDZ^qt; z!^JK(xi^$@vfXOs`c7g$j<;elz=WE#ez|oAGHKAdozFhXZ2FP>{cq3;RsUk=S-1VH z{qV`r&V$d}s)pm;oui%Z;p0vl@2J_2A#E!s&(K~kIBhUBpL0md;hF=Oz5^r>fk$`zhmD zWz$0*39OjP`2z-B8_R|=$0HU9WyBL6Df5L0B-Fq{!t{sZm?<5WRwhi06Xhh+-bBP$ z!IZh53njylpC~h#gn@(%{&>Hg<#T1c@B%4TVMl?5GtE0aWHP;NjS5< z(nlO#OxcwKBA9WdQ(m$Ew^$cP{&QF|=>Z^)qdC;ks_^KrX&oLOh;6!1U=4b8n0KM5 z;j?57I5p_spmoili_kaVv-Jg_IkW`a*sB+C5&Gg0?9yC>+l5y%E9V01adygC42a+2 z@G}H=5&QzdA^>G~yOB5M-LBH%wUh@3z9+Jf$j`(Sa8-D=$$Lw*eGg828uI%XuL&O8 TQehgHX@csSU9uatZX5pvF+SHX diff --git a/docker/docker-executor/app.py b/docker/docker-executor/app.py deleted file mode 100644 index db587df..0000000 --- a/docker/docker-executor/app.py +++ /dev/null @@ -1,94 +0,0 @@ -from fastapi import FastAPI, Request, HTTPException -import shutil -import os -import shlex -import subprocess -import base64 - -# Set environment flag of MAX_EXECUTABLE, MAX_DATA_SIZE - -runtime_version = "docker-executor:0.2.4" -app = FastAPI() - -def get_env(env, flag): - if flag not in env: - raise Exception(flag + " is missing") - return int(env[flag]) - -@app.post("/") -async def execute(request: Request): - env = os.environ.copy() - - MAX_EXECUTABLE = get_env(env, "MAX_EXECUTABLE") - MAX_DATA_SIZE = get_env(env, "MAX_DATA_SIZE") - - request_json = await request.json() - - if "executable" not in request_json: - raise HTTPException(status_code=400, detail="Missing executable value") - executable = base64.b64decode(request_json["executable"]) - if len(executable) > MAX_EXECUTABLE: - raise HTTPException(status_code=400, detail="Executable exceeds max size") - if "calldata" not in request_json: - raise HTTPException(status_code=400, detail="Missing calldata value") - if len(request_json["calldata"]) > MAX_DATA_SIZE: - raise HTTPException(status_code=400, detail="Calldata exceeds max size") - if "timeout" not in request_json: - raise HTTPException(status_code=400, detail="Missing timeout value") - try: - timeout = int(request_json["timeout"]) - except ValueError: - raise HTTPException(status_code=400, detail="Timeout format invalid") - - user_folder = request_json["env"]["BAND_REQUEST_ID"]+"-"+request_json["env"]["BAND_EXTERNAL_ID"] - os.mkdir(user_folder) - path = user_folder+"/execute.sh" - with open(path, "w+") as f: - f.write(executable.decode()) - - os.chmod(path, 0o777) - try: - env = os.environ.copy() - for key, value in request_json.get("env", {}).items(): - env[key] = value - os.environ.update(env) - # proc = subprocess.Popen( - # [path] + shlex.split(request_json["calldata"]), - # env=env, - # stdout=subprocess.PIPE, - # stderr=subprocess.PIPE, - # ) - - # proc.wait(timeout=(timeout / 1000)) - # returncode = proc.returncode - # stdout = proc.stdout.read(MAX_DATA_SIZE).decode() - # stderr = proc.stderr.read(MAX_DATA_SIZE).decode() - returnCode = os.system(path +" "+ ' '.join(shlex.split(request_json["calldata"])) + " > output.txt 2> error.txt") - output = open("output.txt").read(MAX_DATA_SIZE) - error = open("error.txt").read(MAX_DATA_SIZE) - shutil.rmtree(user_folder) - return { - "returncode": returnCode, - "stdout": output, - "stderr": error, - "err": "", - "version": runtime_version, - } - except OSError: - shutil.rmtree(user_folder) - return { - "returncode": 126, - "stdout": "", - "stderr": "", - "err": "Execution fail", - "version": runtime_version, - } - except subprocess.TimeoutExpired: - shutil.rmtree(user_folder) - return { - "returncode": 111, - "stdout": "", - "stderr": "", - "err": "Execution time limit exceeded", - "version": runtime_version, - } diff --git a/docker/docker-executor/requirements.txt b/docker/docker-executor/requirements.txt deleted file mode 100644 index ce33848..0000000 --- a/docker/docker-executor/requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -aiodns==3.0.0 -aiohttp==3.8.1 -aiosignal==1.2.0 -anyio<5,>=3.4.0 -async-timeout==4.0.2 -asynctest==0.13.0 -attrs==21.4.0 -bech32==1.2.0 -ccxt==1.87.1 -certifi==2022.5.18.1 -cffi==1.15.0 -charset-normalizer==2.0.12 -click==8.1.3 -cryptography==37.0.2 -fastapi==0.89.1 -Flask==2.1.2 -frozenlist==1.3.0 -gunicorn==20.1.0 -h11==0.14.0 -idna==3.3 -importlib-metadata==4.11.4 -itsdangerous==2.1.2 -Jinja2==3.1.2 -MarkupSafe==2.1.1 -multidict==6.0.2 -pycares==4.1.2 -pycparser==2.21 -pydantic==1.10.4 -requests==2.28.0 -sniffio>=1.1 -starlette==0.22.0 -typing-extensions==4.4.0 -urllib3==1.26.9 -uvicorn==0.20.0 -websocket-client==1.3.2 -Werkzeug==2.1.2 -yarl==1.7.2 -zipp==3.8.0 diff --git a/docker/executor/app.py b/docker/executor/app.py new file mode 100644 index 0000000..3f9b570 --- /dev/null +++ b/docker/executor/app.py @@ -0,0 +1,76 @@ +from fastapi import FastAPI, Request, HTTPException +import os +import shlex +import base64 +import sys +import io + +# preload +import preload + +runtime_version = "docker-executor:0.2.4" +app = FastAPI() + + +def get_env(env, flag): + if flag not in env: + raise Exception(flag + " is missing") + return int(env[flag]) + + +@app.post("/") +async def execute(request: Request): + base_env = os.environ.copy() + + MAX_EXECUTABLE = get_env(base_env, "MAX_EXECUTABLE") + MAX_DATA_SIZE = get_env(base_env, "MAX_DATA_SIZE") + + request_json = await request.json() + + if "executable" not in request_json: + raise HTTPException(status_code=400, detail="Missing executable value") + executable = base64.b64decode(request_json["executable"]) + if len(executable) > MAX_EXECUTABLE: + raise HTTPException(status_code=400, detail="Executable exceeds max size") + if "calldata" not in request_json: + raise HTTPException(status_code=400, detail="Missing calldata value") + if len(request_json["calldata"]) > MAX_DATA_SIZE: + raise HTTPException(status_code=400, detail="Calldata exceeds max size") + + res = { + "returncode": 0, + "stdout": "", + "stderr": "", + "err": "", + "version": runtime_version, + } + + out = io.StringIO() + err = io.StringIO() + + try: + sys.stdout = out + sys.stderr = err + sys.argv = shlex.split("file " + request_json["calldata"]) + + env = os.environ.copy() + for key, value in request_json.get("env", {}).items(): + env[key] = value + os.environ.update(env) + + exec(executable.decode(), {"__name__": "__main__"}) + except BaseException as e: + res["returncode"] = 126 + res["err"] = "Execution fail" + finally: + os.environ.update(base_env) + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + + res["stdout"] = out.getvalue()[:MAX_DATA_SIZE] + res["stderr"] = err.getvalue()[:MAX_DATA_SIZE] + + if len(res["stderr"]) != 0: + res["returncode"] = 1 + + return res diff --git a/docker/executor/preload.py b/docker/executor/preload.py new file mode 100644 index 0000000..023e030 --- /dev/null +++ b/docker/executor/preload.py @@ -0,0 +1,20 @@ +import aiodns +import aiohttp +import aiosignal +import async_timeout +import attrs +import bech32 +import ccxt +import certifi +import cffi +import charset_normalizer +import cryptography +import frozenlist +import idna +import multidict +import pycares +import pycparser +import requests +import urllib3 +import websocket +import yarl diff --git a/docker/extra_requirements.txt b/docker/extra_requirements.txt new file mode 100644 index 0000000..0611dab --- /dev/null +++ b/docker/extra_requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.89.1 +gunicorn==20.1.0 +uvicorn==0.20.0 diff --git a/scripts/data_source.py b/scripts/data_source.py new file mode 100644 index 0000000..14e6861 --- /dev/null +++ b/scripts/data_source.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import sys +import ccxt +import aiohttp +import asyncio +import os +import time + +if __name__ == "__main__": + try: + print(sys.argv[1:]) + print(os.getpid()) + print(globals()) + except Exception as e: + print(str(e), file=sys.stderr) + sys.exit(1) diff --git a/scripts/load_test.js b/scripts/load_test.js new file mode 100644 index 0000000..ca51042 --- /dev/null +++ b/scripts/load_test.js @@ -0,0 +1,31 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; +import { Counter } from "k6/metrics"; +import { b64encode } from "k6/encoding"; +let ErrorCount = new Counter("errors"); +const data = b64encode(open('./data_source.py')) + +export default function () { + let res = http.post("http://localhost:8000", JSON.stringify({ + "calldata": "TEST_ARG", + "env": { + "BAND_CHAIN_ID": "test-chain-id", + "BAND_EXTERNAL_ID": "test-external-id", + "BAND_REPORTER": "test-reporter", + "BAND_REQUEST_ID": "test-request-id", + "BAND_SIGNATURE": "test-signature", + "BAND_VALIDATOR": "test-validator" + }, + "executable": data, + "timeout": 10000 + }), { headers: { 'Content-Type': 'application/json' } }) + + let success = check(res, { + "status is 200": (r) => r.status === 200, + }); + + if (!success) { + ErrorCount.add(1); + } + +} From c62204ab2c484076be7a95628b74b1e305c29165 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:28:37 +0700 Subject: [PATCH 05/13] adjust wording --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3e18317..959b146 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,4 @@ docker: (cd docker && docker compose up --force-recreate --build -d) rm $(CURRENT_DIR)/docker/requirements.txt $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) - @echo "Your local executor's url is "$(IP)":8000" + @echo "Your local executor's url is http://"$(IP)":8000" From f0d33531f6c0d89fa8a75e718507c275db84de8b Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:32:32 +0700 Subject: [PATCH 06/13] add delay --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 959b146..599cdde 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,5 @@ docker: (cd docker && docker compose up --force-recreate --build -d) rm $(CURRENT_DIR)/docker/requirements.txt $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) + sleep 10 @echo "Your local executor's url is http://"$(IP)":8000" From 8669aedd35a18116e9f994406f0ee7fa30377008 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:33:35 +0700 Subject: [PATCH 07/13] fix order --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 599cdde..d07ea3a 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,6 @@ docker: cat requirements.txt $(CURRENT_DIR)/docker/extra_requirements.txt > $(CURRENT_DIR)/docker/requirements.txt (cd docker && docker compose up --force-recreate --build -d) rm $(CURRENT_DIR)/docker/requirements.txt + sleep 5 $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) - sleep 10 @echo "Your local executor's url is http://"$(IP)":8000" From b1d40d90bd2e37ebd67b4e094394f1ac90710fed Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:37:26 +0700 Subject: [PATCH 08/13] adjust command --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d07ea3a..1b8628d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: lambda gcf docker +.PHONY: lambda gcf docker docker-ip VERSION=2.0.3 CURRENT_DIR=$(shell pwd) @@ -23,6 +23,7 @@ docker: cat requirements.txt $(CURRENT_DIR)/docker/extra_requirements.txt > $(CURRENT_DIR)/docker/requirements.txt (cd docker && docker compose up --force-recreate --build -d) rm $(CURRENT_DIR)/docker/requirements.txt - sleep 5 + +docker-ip: $(eval IP=$(shell docker inspect --format='{{json .NetworkSettings.Networks.docker_default.IPAddress}}' docker-executor-1 | jq .)) @echo "Your local executor's url is http://"$(IP)":8000" From 6e0cb1ad3e635bdd64dfc7569a1029720dbca122 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 01:50:08 +0700 Subject: [PATCH 09/13] try flask to prevent eventloop error from datasource --- docker/Dockerfile | 5 +---- docker/docker-compose.yaml | 4 ++-- docker/executor/app.py | 20 ++++++++++++-------- docker/extra_requirements.txt | 3 +-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1af62c5..c9f5dba 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,4 @@ EXPOSE 8000 RUN adduser --system --no-create-home nonroot USER nonroot -CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "-k", "uvicorn.workers.UvicornWorker", "--workers", "32", "--timeout", "0"] - -# FOR TESTING ONLY -# CMD ["uvicorn","app:app", "--reload", "--host", "0.0.0.0", "--port", "8000"] +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "32", "--timeout", "0"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 653af6f..9a75fa0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -4,8 +4,8 @@ services: build: context: . restart: always + ports: + - "8000:8000" # FOR TESTING ONLY - # ports: - # - "8000:8000" # volumes: # - ./executor:/executor diff --git a/docker/executor/app.py b/docker/executor/app.py index 3f9b570..675f03e 100644 --- a/docker/executor/app.py +++ b/docker/executor/app.py @@ -1,4 +1,4 @@ -from fastapi import FastAPI, Request, HTTPException +from flask import jsonify, Flask, request import os import shlex import base64 @@ -9,7 +9,7 @@ import preload runtime_version = "docker-executor:0.2.4" -app = FastAPI() +app = Flask(__name__) def get_env(env, flag): @@ -18,24 +18,28 @@ def get_env(env, flag): return int(env[flag]) +def bad_request(err): + return jsonify({"error": err}), 400 + + @app.post("/") -async def execute(request: Request): +def execute(): base_env = os.environ.copy() MAX_EXECUTABLE = get_env(base_env, "MAX_EXECUTABLE") MAX_DATA_SIZE = get_env(base_env, "MAX_DATA_SIZE") - request_json = await request.json() + request_json = request.get_json(force=True) if "executable" not in request_json: - raise HTTPException(status_code=400, detail="Missing executable value") + raise bad_request("Missing executable value") executable = base64.b64decode(request_json["executable"]) if len(executable) > MAX_EXECUTABLE: - raise HTTPException(status_code=400, detail="Executable exceeds max size") + raise bad_request("Executable exceeds max size") if "calldata" not in request_json: - raise HTTPException(status_code=400, detail="Missing calldata value") + raise bad_request("Missing calldata value") if len(request_json["calldata"]) > MAX_DATA_SIZE: - raise HTTPException(status_code=400, detail="Calldata exceeds max size") + raise bad_request("Calldata exceeds max size") res = { "returncode": 0, diff --git a/docker/extra_requirements.txt b/docker/extra_requirements.txt index 0611dab..70c13bf 100644 --- a/docker/extra_requirements.txt +++ b/docker/extra_requirements.txt @@ -1,3 +1,2 @@ -fastapi==0.89.1 +Flask==2.2.2 gunicorn==20.1.0 -uvicorn==0.20.0 From b50d4cebb8705f3195786735d9ef780191967419 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 02:25:48 +0700 Subject: [PATCH 10/13] add timeout --- docker/Dockerfile | 2 +- scripts/data_source.py | 1 + scripts/load_test.js | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c9f5dba..e1dcb3a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,4 +15,4 @@ EXPOSE 8000 RUN adduser --system --no-create-home nonroot USER nonroot -CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "32", "--timeout", "0"] +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "32", "--timeout", "10"] diff --git a/scripts/data_source.py b/scripts/data_source.py index 14e6861..55e0179 100644 --- a/scripts/data_source.py +++ b/scripts/data_source.py @@ -9,6 +9,7 @@ if __name__ == "__main__": try: + time.sleep(5) print(sys.argv[1:]) print(os.getpid()) print(globals()) diff --git a/scripts/load_test.js b/scripts/load_test.js index ca51042..879d401 100644 --- a/scripts/load_test.js +++ b/scripts/load_test.js @@ -20,6 +20,8 @@ export default function () { "timeout": 10000 }), { headers: { 'Content-Type': 'application/json' } }) + console.log(res.body) + let success = check(res, { "status is 200": (r) => r.status === 200, }); From ed74967aa8c683492a1e343eb99b48825afc8b0e Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Feb 2023 03:47:07 +0700 Subject: [PATCH 11/13] adjust worker --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e1dcb3a..76a45f8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,4 +15,4 @@ EXPOSE 8000 RUN adduser --system --no-create-home nonroot USER nonroot -CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "32", "--timeout", "10"] +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "16", "--timeout", "10"] From 87e4aa6f2eb8cf02912a5629b2f5dd4fc98bf985 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Mon, 6 Mar 2023 23:51:07 +0700 Subject: [PATCH 12/13] for testing --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 76a45f8..6901666 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,4 +15,4 @@ EXPOSE 8000 RUN adduser --system --no-create-home nonroot USER nonroot -CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "16", "--timeout", "10"] +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "1", "--timeout", "10"] From 75c60bdc38d7abb7f044a9a13fe82d63fc367578 Mon Sep 17 00:00:00 2001 From: Kitipong Sirirueangsakul Date: Tue, 7 Mar 2023 03:55:06 +0700 Subject: [PATCH 13/13] set worker as 16 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6901666..76a45f8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,4 +15,4 @@ EXPOSE 8000 RUN adduser --system --no-create-home nonroot USER nonroot -CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "1", "--timeout", "10"] +CMD ["gunicorn", "app:app", "--bind", "0.0.0.0:8000", "--workers", "16", "--timeout", "10"]