Merge pull request #1447 from maaquib/master

msaroufim · web-flow · commit a093a0d1f5e6 · 2022-02-20T13:49:41.000-08:00
Updating buildspecs related resources to py38
diff --git a/ci/buildspec_cpu_backup.yml b/ci/buildspec_cpu_backup.yml
diff --git a/ci/buildspec_gpu_backup.yml b/ci/buildspec_gpu_backup.yml
diff --git a/ci/launcher/launch_test.py b/ci/launcher/launch_test.py
@@ -15,14 +15,14 @@
 from utils import ec2 as ec2_utils
 
 CPU_INSTANCE_COMMANDS_LIST = [
-    "python3 ts_scripts/install_dependencies.py --environment=dev",
-    "python3 torchserve_sanity.py",
+    "python ts_scripts/install_dependencies.py --environment=dev",
+    "python torchserve_sanity.py",
     "cd serving-sdk/ && mvn clean install -q && cd ../",
 ]
 
 GPU_INSTANCE_COMMANDS_LIST = [
-    "python3 ts_scripts/install_dependencies.py --environment=dev --cuda=cu102",
-    "python3 torchserve_sanity.py",
+    "python ts_scripts/install_dependencies.py --environment=dev --cuda=cu102",
+    "python torchserve_sanity.py",
     "cd serving-sdk/ && mvn clean install -q && cd ../",
 ]
 
@@ -38,7 +38,7 @@ def run_commands_on_ec2_instance(ec2_connection, is_gpu):
     virtual_env_name = "venv"
 
     with ec2_connection.cd(f"/home/ubuntu/serve"):
-        ec2_connection.run(f"python3 -m venv {virtual_env_name}")
+        ec2_connection.run(f"python -m venv {virtual_env_name}")
         with ec2_connection.prefix(f"source {virtual_env_name}/bin/activate"):
             commands_list = GPU_INSTANCE_COMMANDS_LIST if is_gpu else CPU_INSTANCE_COMMANDS_LIST
 
@@ -128,14 +128,13 @@ def launch_ec2_instance(region, instance_type, ami_id):
             else:
                 ec2_connection.run(f"cd serve && git fetch origin {github_pull_request_number}")
 
-            ec2_connection.run(f"sudo apt-get install -y python3-venv")
+            ec2_connection.run(f"sudo apt-get install -y python3.8-dev python3.8-distutils python3.8-venv")
+            # update alternatives doesn't seem to work
+            ec2_connection.run(f"sudo ln -fs /usr/bin/python3.8 /usr/bin/python")
             # Following is necessary on Base Ubuntu DLAMI because the default python is python2
             # This will NOT fail for other AMI where default python is python3
             ec2_connection.run(
-                f"sudo cp /usr/local/bin/pip3 /usr/local/bin/pip && pip install --upgrade pip", warn=True
-            )
-            ec2_connection.run(
-                f"sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1", warn=True
+                f"curl -O https://bootstrap.pypa.io/get-pip.py && python get-pip.py", warn=True
             )
 
         is_gpu = True if instance_type[:2] in GPU_INSTANCES else False
@@ -197,3 +196,4 @@ def main():
 
 if __name__ == "__main__":
     main()
+
diff --git a/requirements/common.txt b/requirements/common.txt
@@ -1,9 +1,9 @@
 future
 psutil
+cython
 wheel
 requests
-sentencepiece
-pillow==8.3.2
+pillow==9.0.0
 captum
 packaging
 numpy; sys_platform != 'win32'
diff --git a/requirements/torch_cu101_linux.txt b/requirements/torch_cu101_linux.txt
@@ -1,6 +1,9 @@
 #pip install torch==1.8.1+cu101 torchvision==0.9.1+cu101 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+wheel
+pillow==9.0.0
 torch==1.8.1+cu101; sys_platform == 'linux'
 torchvision==0.9.1+cu101; sys_platform == 'linux'
 torchtext==0.9.1; sys_platform == 'linux'
-torchaudio==0.8.1; sys_platform == 'linux'
+torchaudio==0.8.1; sys_platform == 'linux'
diff --git a/requirements/torch_cu102_linux.txt b/requirements/torch_cu102_linux.txt
@@ -1,4 +1,7 @@
 # pip install torch torchvision
+cython
+wheel
+pillow==9.0.0
 torch==1.10.0; sys_platform == 'linux'
 torchvision==0.11.1; sys_platform == 'linux'
 torchtext==0.11.0; sys_platform == 'linux'
diff --git a/requirements/torch_cu111_linux.txt b/requirements/torch_cu111_linux.txt
@@ -1,6 +1,9 @@
 #pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 torchaudio===0.9.0 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+wheel
+pillow==9.0.0
 torch==1.9.0+cu111; sys_platform == 'linux'
 torchvision==0.10.0+cu111; sys_platform == 'linux'
 torchtext==0.10.0; sys_platform == 'linux'
-torchaudio==0.9.0; sys_platform == 'linux'
+torchaudio==0.9.0; sys_platform == 'linux'
diff --git a/requirements/torch_cu113_linux.txt b/requirements/torch_cu113_linux.txt
@@ -1,5 +1,8 @@
 #pip3 install torch==1.10.0+cu113 torchvision==0.11.1+cu113 torchaudio==0.10.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+wheel
+pillow==9.0.0
 torch==1.10.0+cu113; sys_platform == 'linux'
 torchvision==0.11.1+cu113; sys_platform == 'linux'
 torchtext==0.11.0; sys_platform == 'linux'
diff --git a/requirements/torch_cu92_linux.txt b/requirements/torch_cu92_linux.txt
@@ -1,6 +1,9 @@
 #pip install torch==1.8.1+cu92 torchvision==0.9.1+cu92 torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html
 --find-links https://download.pytorch.org/whl/torch_stable.html
+cython
+wheel
+pillow==9.0.0
 torch==1.8.1+cu92; sys_platform == 'linux'
 torchvision==0.9.1+cu92; sys_platform == 'linux'
 torchtext==0.9.1; sys_platform == 'linux'
-torchaudio==0.8.1; sys_platform == 'linux'
+torchaudio==0.8.1; sys_platform == 'linux'
diff --git a/requirements/torch_linux.txt b/requirements/torch_linux.txt
@@ -1,5 +1,8 @@
 #pip3 install torch==1.10.0+cpu torchvision==0.11.1+cpu torchaudio==0.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html
 --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+cython
+wheel
+pillow==9.0.0
 torch==1.10.0+cpu; sys_platform == 'linux'
 torchvision==0.11.1+cpu; sys_platform == 'linux'
 torchtext==0.11.0; sys_platform == 'linux'
diff --git a/requirements/torch_windows.txt b/requirements/torch_windows.txt
@@ -1,5 +1,7 @@
-pillow==8.3.2
+wheel
 torch==1.10.0; sys_platform == 'win32'
 torchvision==0.11.1; sys_platform == 'win32'
 torchtext==0.11.0; sys_platform == 'win32'
-torchaudio==0.10.0; sys_platform == 'win32'
+torchaudio==0.10.0; sys_platform == 'win32'
+pillow==9.0.0
+
diff --git a/setup.py b/setup.py
@@ -157,7 +157,7 @@ def run(self):
 
     print(f"-- {name} building version: {version}")
 
-    requirements = ['Pillow', 'psutil', 'future', 'packaging']
+    requirements = ['Pillow', 'psutil', 'future', 'packaging', 'wheel']
 
     setup(
         name=name,
diff --git a/test/buildspec.yml b/test/buildspec.yml
@@ -4,9 +4,13 @@ version: 0.2
 
 phases:
   install:
+    runtime-versions:
+      - python: 3.8
     commands:
       - apt-get update
       - apt-get install sudo -y
+      - curl -O https://bootstrap.pypa.io/get-pip.py && python get-pip.py
+      - update-alternatives --install /usr/local/bin/pip pip /usr/local/bin/pip3 1
       - python ts_scripts/install_dependencies.py --environment=dev
 
   build:
@@ -16,4 +20,4 @@ phases:
 artifacts:
   files:
     - /tmp/test_exec.log
-  name: TS-NIGHTLY-REGRESSION-$(date +%Y-%m-%d)
+  name: TS-NIGHTLY-REGRESSION-$(date +%Y-%m-%d)
diff --git a/test/buildspec_gpu.yml b/test/buildspec_gpu.yml
@@ -3,10 +3,14 @@
 version: 0.2
 
 phases:
+    runtime-versions:
+      - python: 3.8
   install:
     commands:
       - apt-get update
       - apt-get install sudo -y
+      - curl -O https://bootstrap.pypa.io/get-pip.py && python get-pip.py
+      - update-alternatives --install /usr/local/bin/pip pip /usr/local/bin/pip3 1
       - python ts_scripts/install_dependencies.py --cuda=cu102 --environment=dev
 
   build:
diff --git a/ts_scripts/install_dependencies.py b/ts_scripts/install_dependencies.py
@@ -37,11 +37,11 @@ def install_torch_packages(self, cuda_version):
                 sys.exit(1)
             else:
                 os.system(
-                    f"pip install -U -r requirements/torch_{cuda_version}_{platform.system().lower()}.txt"
+                    f"python -m pip install -U -r requirements/torch_{cuda_version}_{platform.system().lower()}.txt"
                 )
         else:
             os.system(
-                f"pip install -U -r requirements/torch_{platform.system().lower()}.txt"
+                f"python -m pip install -U -r requirements/torch_{platform.system().lower()}.txt"
             )
 
     def install_python_packages(self, cuda_version, requirements_file_path):
@@ -51,9 +51,9 @@ def install_python_packages(self, cuda_version, requirements_file_path):
             os.system("conda install -y conda-build")
 
         self.install_torch_packages(cuda_version)
-        os.system("pip install -U pip setuptools")
+        os.system("python -m pip install -U pip setuptools")
         # developer.txt also installs packages from common.txt
-        os.system("pip install -U -r {0}".format(requirements_file_path))
+        os.system("python -m pip install -U -r {0}".format(requirements_file_path))
         # If conda is available install conda-build package
 
     def install_node_packages(self):
@@ -72,22 +72,22 @@ class Linux(Common):
 
     def __init__(self):
         super().__init__()
-        os.system(f"{self.sudo_cmd}apt-get update")
+        if args.force:
+            os.system(f"{self.sudo_cmd}apt-get update")
 
     def install_java(self):
-        os.system(f"{self.sudo_cmd}apt-get install -y openjdk-11-jdk")
+        if os.system("javac --version") != 0 or args.force:
+            os.system(f"{self.sudo_cmd}apt-get install -y openjdk-11-jdk")
 
     def install_nodejs(self):
-        python_path = Path(sys.executable).resolve()
-        os.system(
-            f"{self.sudo_cmd}curl -sL https://deb.nodesource.com/setup_14.x | {self.sudo_cmd}bash -"
-        )
-        os.system(f"{self.sudo_cmd}apt-get install -y nodejs")
-        os.system(f"{self.sudo_cmd}ln -sf {python_path} /usr/bin/python")
-        os.system(f"{self.sudo_cmd}ln -sf /usr/bin/pip3 /usr/bin/pip")
+        if os.system("node") != 0 or args.force:
+            # os.system(f"{self.sudo_cmd}curl -sL https://deb.nodesource.com/setup_14.x | {self.sudo_cmd}bash -")
+            os.system(f"{self.sudo_cmd}apt-get install -y nodejs")
+
 
     def install_wget(self):
-        os.system(f"{self.sudo_cmd}apt-get install -y wget")
+        if os.system("wget --version") != 0 or args.force:
+            os.system(f"{self.sudo_cmd}apt-get install -y wget")
 
     def install_libgit2(self):
         os.system(
@@ -125,15 +125,16 @@ def __init__(self):
         super().__init__()
 
     def install_java(self):
-        out = get_brew_version()
-        if out == "N/A":
-            sys.exit("**Error: Homebrew not installed...")
-
-        os.system("brew tap AdoptOpenJDK/openjdk")
-        if out >= "2.7":
-            os.system("brew install --cask adoptopenjdk11")
-        else:
-            os.system("brew cask install adoptopenjdk11")
+        if os.system("javac -version") != 0 or args.force:
+            out = get_brew_version()
+            if out == "N/A":
+                sys.exit("**Error: Homebrew not installed...")
+
+            os.system("brew tap AdoptOpenJDK/openjdk")
+            if out >= "2.7":
+                os.system("brew install --cask adoptopenjdk11")
+            else:
+                os.system("brew cask install adoptopenjdk11")
 
     def install_nodejs(self):
         os.system("brew unlink node")
@@ -144,7 +145,8 @@ def install_node_packages(self):
         os.system(f"{self.sudo_cmd} ./ts_scripts/mac_npm_deps")
 
     def install_wget(self):
-        os.system("brew install wget")
+        if os.system("wget --version") != 0 or args.force:
+            os.system("brew install wget")
 
 
 def install_dependencies(cuda_version=None):
@@ -175,20 +177,11 @@ def get_brew_version():
 
 if __name__ == "__main__":
     check_python_version()
-    parser = argparse.ArgumentParser(
-        description="Install various build and test dependencies of TorchServe")
-    parser.add_argument('--cuda',
-                        default=None,
-                        choices=['cu92', 'cu101', 'cu102', 'cu111', 'cu113'],
-                        help="CUDA version for torch")
-    parser.add_argument(
-        '--environment',
-        default='prod',
-        choices=['prod', 'dev'],
-        help=
-        "environment(production or developer) on which dependencies will be installed"
-    )
-
+    parser = argparse.ArgumentParser(description="Install various build and test dependencies of TorchServe")
+    parser.add_argument('--cuda', default=None, choices=['cu92', 'cu101', 'cu102', 'cu111'], help="CUDA version for torch")
+    parser.add_argument('--environment', default='prod', choices=['prod', 'dev'],
+                        help="environment(production or developer) on which dependencies will be installed")
+    parser.add_argument("--force", action='store_true', help="force reinstall dependencies wget, node, java and apt-update")
     args = parser.parse_args()
 
-    install_dependencies(cuda_version=args.cuda)
+    install_dependencies(cuda_version=args.cuda)
diff --git a/ts_scripts/setup_wsl_ubuntu b/ts_scripts/setup_wsl_ubuntu
@@ -2,11 +2,11 @@
 sudo apt-get update
 sudo apt-get install -y \
 g++ \
-python3-dev \
+python3.8-dev \
 && cd /tmp \
 && curl -O https://bootstrap.pypa.io/get-pip.py \
-&& python3 get-pip.py \
+&& python3.8 get-pip.py \
 && cd -
 
-sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
 sudo update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip3 1
