build+ci+jenkins: make local, ci, and deploy envs identical

dshemetov · dshemetov · commit 8a72ca0b7a75 · 2024-08-09T10:33:40.000-07:00
* pin pip==23.0.1 everywhere
* remove pip install calls from ci and deploy,
  make sure all dependencies are listed in
  pyproject.toml or setup.py and installed by the
  Makefile
* remove wheel from dependencies, only needed
  wheel for bdist_wheel command, which
  setuptools&gt;=70.1 provides
diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
@@ -56,7 +56,7 @@ jobs:
         if: ${{ steps.changes.outputs.utils == 'true' }}
         run: git tag delphi-utils/v${{ steps.utils.outputs.version }}
       - name: Publish tags
-        run: git push --tags 
+        run: git push --tags
     outputs:
       indicators-version: ${{ steps.indicators.outputs.version }}
       utils-version: ${{ steps.utils.outputs.version }}
@@ -76,10 +76,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.8
-      - name: Install testing dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pylint pytest pydocstyle wheel twine
       - name: Install
         run: |
           make install
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -52,10 +52,6 @@ jobs:
           python-version: 3.8
           cache: "pip"
           cache-dependency-path: "pyproject.toml"
-      - name: Install testing dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pylint pytest pydocstyle wheel
       - name: Install
         run: |
           make install-ci
diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile
@@ -1,16 +1,16 @@
 .PHONY = venv, lint, test, clean, release
 
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e '.[dev]'
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install 'build[virtualenv]' pylint pytest pydocstyle wheel twine ; \
 	pip install '.[dev]'
 
 lint:
@@ -30,5 +30,4 @@ clean:
 
 release: lint test
 	. env/bin/activate ; \
-	pip install 'build[virtualenv]' ; \
-	python -m build --sdist --wheel
+	python -m build --sdist --wheel
diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml
@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "setuptools-scm>=8.0"]
+requires = ["setuptools>=70.1", "setuptools-scm>=8.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -36,6 +36,7 @@ Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
 
 [project.optional-dependencies]
 dev = [
+    "build[virtualenv]",
     "darker[isort]~=2.1.1",
     "pylint==2.8.3",
     "pytest",
@@ -45,6 +46,7 @@ dev = [
     "moto~=4.2.14",
     "requests-mock",
     "freezegun",
+    "twine"
 ]
 flash = ["scipy"]
 
diff --git a/_template_python/Makefile b/_template_python/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile
@@ -56,7 +56,7 @@ install-python:
 	fi
 	python3 -m venv env
 	source env/bin/activate && \
-	python -m pip install --upgrade pip && \
+	python -m pip install pip==23.0.1 && \
 	pip install wheel && \
 	pip install --timeout 1000 delphi_utils
 
diff --git a/changehc/Makefile b/changehc/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/claims_hosp/Makefile b/claims_hosp/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/claims_hosp/setup.py b/claims_hosp/setup.py
@@ -6,6 +6,7 @@
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "numpy",
+    "pydocstyle",
     "pandas",
     "paramiko",
     "pyarrow",
diff --git a/doctor_visits/Makefile b/doctor_visits/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/doctor_visits/setup.py b/doctor_visits/setup.py
@@ -5,6 +5,7 @@
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "numpy",
+    "pydocstyle",
     "pandas",
     "paramiko",
     "pylint==2.8.3",
diff --git a/google_symptoms/Makefile b/google_symptoms/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/google_symptoms/setup.py b/google_symptoms/setup.py
@@ -8,6 +8,7 @@
     "freezegun",
     "mock",
     "numpy",
+    "pydocstyle",
     "pandas-gbq",
     "pandas",
     "pydocstyle",
diff --git a/hhs_hosp/Makefile b/hhs_hosp/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/jenkins/build-indicator.sh b/jenkins/build-indicator.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Jenkins build 
+# Jenkins build
 #
 
 set -eo pipefail
@@ -14,7 +14,6 @@ cd "${WORKSPACE}/${local_indicator}" || exit
 # Set up venv
 python -m venv env
 source env/bin/activate
-pip install pip==23.0.1 --retries 10 --timeout 20
-pip install numpy --retries 10 --timeout 20
+python -m pip install pip==23.0.1 --retries 10 --timeout 20
 pip install ../_delphi_utils_python/. --retries 10 --timeout 20
 [ ! -f setup.py ] || pip install . --retries 10 --timeout 20
diff --git a/nchs_mortality/Makefile b/nchs_mortality/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/nssp/Makefile b/nssp/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/nwss_wastewater/Makefile b/nwss_wastewater/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/quidel_covidtest/Makefile b/quidel_covidtest/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/sir_complainsalot/Makefile b/sir_complainsalot/Makefile
@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 
diff --git a/sir_complainsalot/setup.py b/sir_complainsalot/setup.py
@@ -6,6 +6,7 @@
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "pandas",
+    "pydocstyle",
     "pylint==2.8.3",
     "pytest-cov",
     "pytest",
