Merge pull request #113 from Labelbox/develop

repair-tests

Author: msokoloff1

.github/workflows/python-package.yml

@@ -67,7 +67,7 @@ jobs:
       env:
         # make sure to tell tox to use these environs in tox.ini
         #
-        # randall@labelbox.com
+        # msokoloff+prod-python@labelbox.com
         LABELBOX_TEST_API_KEY_PROD: ${{ secrets.LABELBOX_API_KEY }}
 
         # randall+staging-python@labelbox.com

CHANGELOG.md

@@ -6,7 +6,12 @@
 * Comparing a Labelbox object (e.g. Project) to None doesn't raise an exception
 * Adding `order_by` to `Project.labels` doesn't raise an exception
 
-## Version 2.4.10 (2021-01-05)
+## Version 2.4.11 (2021-03-07)
+### Fix
+* Increase query timeout
+* Retry 502s
+
+## Version 2.4.10 (2021-02-05)
 ### Added 
 * SDK version added to request headers
 

Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.6
+FROM python:3.7
+
+RUN pip install pytest
 
 COPY . /usr/src/labelbox
 WORKDIR /usr/src/labelbox
 
-RUN pip install pytest
 RUN python setup.py install

Makefile

@@ -5,11 +5,11 @@ build:
 test-staging: build
 	docker run -it -v ${PWD}:/usr/src -w /usr/src \
 		-e LABELBOX_TEST_ENVIRON="staging" \
-		-e LABELBOX_TEST_API_KEY_STAGING="<REPLACE>" \
+		-e LABELBOX_TEST_API_KEY_STAGING=${LABELBOX_TEST_API_KEY_STAGING} \
 		local/labelbox-python:test pytest $(PATH_TO_TEST) -svvx
 
 test-prod: build
 	docker run -it -v ${PWD}:/usr/src -w /usr/src \
 		-e LABELBOX_TEST_ENVIRON="prod" \
-		-e LABELBOX_TEST_API_KEY_PROD="<REPLACE>" \
+		-e LABELBOX_TEST_API_KEY_PROD=${LABELBOX_TEST_API_KEY_PROD} \
 		local/labelbox-python:test pytest $(PATH_TO_TEST) -svvx

labelbox/__init__.py

@@ -1,5 +1,5 @@
 name = "labelbox"
-__version__ = "2.4.10"
+__version__ = "2.4.11"
 
 from labelbox.client import Client
 from labelbox.schema.bulk_import_request import BulkImportRequest
@@ -15,4 +15,4 @@
 from labelbox.schema.asset_metadata import AssetMetadata
 from labelbox.schema.webhook import Webhook
 from labelbox.schema.prediction import Prediction, PredictionModel
-from labelbox.schema.ontology import Ontology
+from labelbox.schema.ontology import Ontology

labelbox/client.py

@@ -71,9 +71,10 @@ def __init__(self,
             'X-User-Agent': f'python-sdk {SDK_VERSION}'
         }
 
+    #TODO: Add exponential backoff so we don'tt overwhelm the api
     @retry.Retry(predicate=retry.if_exception_type(
         labelbox.exceptions.InternalServerError))
-    def execute(self, query, params=None, timeout=10.0):
+    def execute(self, query, params=None, timeout=30.0):
         """ Sends a request to the server for the execution of the
         given query.
 
@@ -126,25 +127,23 @@ def convert_value(value):
             logger.debug("Response: %s", response.text)
         except requests.exceptions.Timeout as e:
             raise labelbox.exceptions.TimeoutError(str(e))
-
         except requests.exceptions.RequestException as e:
             logger.error("Unknown error: %s", str(e))
             raise labelbox.exceptions.NetworkError(e)
-
         except Exception as e:
             raise labelbox.exceptions.LabelboxError(
                 "Unknown error during Client.query(): " + str(e), e)
-
         try:
             r_json = response.json()
         except:
-            error_502 = '502 Bad Gateway'
-            if error_502 in response.text:
-                raise labelbox.exceptions.InternalServerError(error_502)
             if "upstream connect error or disconnect/reset before headers" \
                     in response.text:
                 raise labelbox.exceptions.InternalServerError(
                     "Connection reset")
+            elif response.status_code == 502:
+                error_502 = '502 Bad Gateway'
+                raise labelbox.exceptions.InternalServerError(error_502)
+
             raise labelbox.exceptions.LabelboxError(
                 "Failed to parse response as JSON: %s" % response.text)
 
@@ -189,6 +188,7 @@ def check_errors(keywords, *path):
 
         # Check if API limit was exceeded
         response_msg = r_json.get("message", "")
+
         if response_msg.startswith("You have exceeded"):
             raise labelbox.exceptions.ApiLimitError(response_msg)
 
@@ -292,7 +292,6 @@ def upload_data(self,
                 "1": (filename, content, content_type) if
                      (filename and content_type) else content
             })
-
         try:
             file_data = response.json().get("data", None)
         except ValueError as e:  # response is not valid JSON

labelbox/schema/ontology.py

@@ -66,7 +66,6 @@ def from_json(cls, json_dict):
 class Ontology(DbObject):
     """An ontology specifies which tools and classifications are available
     to a project. This is read only for now.
-
     Attributes:
         name (str)
         description (str)
@@ -75,7 +74,6 @@ class Ontology(DbObject):
         normalized (json)
         object_schema_count (int)
         classification_schema_count (int)
-
         projects (Relationship): `ToMany` relationship to Project
         created_by (Relationship): `ToOne` relationship to User
     """

tests/integration/test_client_errors.py

@@ -1,7 +1,6 @@
 from multiprocessing.dummy import Pool
 import os
 import time
-
 import pytest
 
 from labelbox import Project, Dataset, User
@@ -111,21 +110,24 @@ def test_invalid_attribute_error(client, rand_gen):
     project.delete()
 
 
-@pytest.mark.slow
-# TODO improve consistency
-@pytest.mark.skip(reason="Inconsistent test")
-def test_api_limit_error(client, rand_gen):
-    project_id = client.create_project(name=rand_gen(str)).uid
+@pytest.mark.skip("timeouts cause failure before rate limit")
+def test_api_limit_error(client):
 
     def get(arg):
         try:
-            return client.get_project(project_id)
+            return client.get_user()
         except labelbox.exceptions.ApiLimitError as e:
             return e
 
-    with Pool(300) as pool:
-        results = pool.map(get, list(range(2000)))
+    #Rate limited at 1500 + buffer
+    n = 1600
+    #max of 30 concurrency before the service becomes unavailable
+    with Pool(30) as pool:
+        start = time.time()
+        results = list(pool.imap(get, range(n)), total=n)
+        elapsed = time.time() - start
 
+    assert elapsed < 60, "Didn't finish fast enough"
     assert labelbox.exceptions.ApiLimitError in {type(r) for r in results}
 
     # Sleep at the end of this test to allow other tests to execute.

tests/integration/test_data_upload.py

@@ -2,10 +2,8 @@
 import requests
 
 
-# TODO it seems that at some point Google Storage (gs prefix) started being
-# returned, and we can't just download those with requests. Fix this
-@pytest.mark.skip
-def test_file_upload(client, rand_gen):
+def test_file_upload(client, rand_gen, dataset):
     data = rand_gen(str)
-    url = client.upload_data(data.encode())
-    assert requests.get(url).text == data
+    uri = client.upload_data(data.encode())
+    data_row = dataset.create_data_row(row_data=uri)
+    assert requests.get(data_row.row_data).text == data

tests/integration/test_label.py

@@ -1,3 +1,4 @@
+from labelbox.schema.labeling_frontend import LabelingFrontend
 import time
 
 import pytest
@@ -30,18 +31,22 @@ def test_labels(label_pack):
     assert list(data_row.labels()) == []
 
 
-# TODO check if this is supported or not
-@pytest.mark.skip
-def test_label_export(label_pack):
+def test_label_export(client, label_pack):
     project, dataset, data_row, label = label_pack
-    project.create_label(data_row=data_row, label="l2")
-
-    exported_labels_url = project.export_labels(5)
+    #Old create_label works even with projects setup using the new editor.
+    #It will appear in the export, just not in the new editor
+    project.create_label(data_row=data_row, label="export_label")
+    #Project has to be setup for export to be possible
+    editor = list(
+        client.get_labeling_frontends(
+            where=LabelingFrontend.name == "editor"))[0]
+    empty_ontology = {"tools": [], "classifications": []}
+    project.setup(editor, empty_ontology)
+    exported_labels_url = project.export_labels()
     assert exported_labels_url is not None
-    if exported_labels_url is not None:
-        exported_labels = requests.get(exported_labels_url)
-        # TODO check content
-        assert False
+    exported_labels = requests.get(exported_labels_url)
+    labels = [example['Label'] for example in exported_labels.json()]
+    assert 'export_label' in labels
 
 
 def test_label_update(label_pack):