Skip to content

Commit

Permalink
Merge branch 'master' into refactor/spawn/setup-environment
Browse files Browse the repository at this point in the history
  • Loading branch information
Borda committed Sep 12, 2022
2 parents cd9bf08 + e5998e6 commit 1040d35
Show file tree
Hide file tree
Showing 959 changed files with 13,975 additions and 7,334 deletions.
7 changes: 6 additions & 1 deletion .actions/assistant.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import datetime
import glob
import json
import logging
import os
import re
import shutil
Expand All @@ -16,6 +17,7 @@

import fire
import pkg_resources
from packaging.version import parse as version_parse

REQUIREMENT_FILES = {
"pytorch": (
Expand Down Expand Up @@ -123,14 +125,17 @@ def download_package(package: str, folder: str = ".", version: Optional[str] = N
data = json.load(urlopen(Request(url)))
if not version:
versions = list(data["releases"].keys())
version = sorted(versions, key=LooseVersion)[-1]
versions = sorted(versions, key=lambda x: version_parse(x))
logging.debug(f"Available versions: {versions}")
version = versions[-1]
releases = list(filter(lambda r: r["packagetype"] == "sdist", data["releases"][version]))
assert releases, f"Missing 'sdist' for this package/version aka {package}/{version}"
release = releases[0]
pkg_url = release["url"]
pkg_file = os.path.basename(pkg_url)
pkg_path = os.path.join(folder, pkg_file)
os.makedirs(folder, exist_ok=True)
print(f"downloading: {pkg_url}")
request.urlretrieve(pkg_url, pkg_path)

@staticmethod
Expand Down
3 changes: 3 additions & 0 deletions .actions/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
fire
packaging
requests
121 changes: 92 additions & 29 deletions .actions/setup_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,14 @@
import tempfile
import urllib.request
from datetime import datetime
from distutils.version import LooseVersion
from importlib.util import module_from_spec, spec_from_file_location
from itertools import chain, groupby
from types import ModuleType
from typing import List

from pkg_resources import parse_requirements

_PROJECT_ROOT = os.path.dirname(os.path.dirname(__file__))
_PACKAGE_MAPPING = {"pytorch": "pytorch_lightning", "app": "lightning_app"}

Expand All @@ -42,45 +45,92 @@ def _load_py_module(name: str, location: str) -> ModuleType:
return py


def _augment_requirement(ln: str, comment_char: str = "#", unfreeze: str = "all") -> str:
"""Adjust the upper version contrains.
Args:
ln: raw line from requirement
comment_char: charter marking comment
unfreeze: Enum or "all"|"major"|""
Returns:
adjusted requirement
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # anything", unfreeze="")
'arrow>=1.2.0, <=1.2.2'
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # strict", unfreeze="")
'arrow>=1.2.0, <=1.2.2 # strict'
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # my name", unfreeze="all")
'arrow>=1.2.0'
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # strict", unfreeze="all")
'arrow>=1.2.0, <=1.2.2 # strict'
>>> _augment_requirement("arrow", unfreeze="all")
'arrow'
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # cool", unfreeze="major")
'arrow>=1.2.0, <2.0 # strict'
>>> _augment_requirement("arrow>=1.2.0, <=1.2.2 # strict", unfreeze="major")
'arrow>=1.2.0, <=1.2.2 # strict'
>>> _augment_requirement("arrow>=1.2.0", unfreeze="major")
'arrow>=1.2.0, <2.0 # strict'
>>> _augment_requirement("arrow", unfreeze="major")
'arrow'
"""
# filer all comments
if comment_char in ln:
comment = ln[ln.index(comment_char) :]
ln = ln[: ln.index(comment_char)]
is_strict = "strict" in comment
else:
is_strict = False
req = ln.strip()
# skip directly installed dependencies
if not req or req.startswith("http") or "@http" in req:
return ""
# extract the major version from all listed versions
if unfreeze == "major":
req_ = list(parse_requirements([req]))[0]
vers = [LooseVersion(v) for s, v in req_.specs if s not in ("==", "~=")]
ver_major = sorted(vers)[-1].version[0] if vers else None
else:
ver_major = None

# remove version restrictions unless they are strict
if unfreeze and "<" in req and not is_strict:
req = re.sub(r",? *<=? *[\d\.\*]+", "", req).strip()
if ver_major is not None and not is_strict:
# add , only if there are already some versions
req += f"{',' if any(c in req for c in '<=>') else ''} <{int(ver_major) + 1}.0"

# adding strict back to the comment
if is_strict or ver_major is not None:
req += " # strict"

return req


def load_requirements(
path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: bool = True
path_dir: str, file_name: str = "base.txt", comment_char: str = "#", unfreeze: str = "all"
) -> List[str]:
"""Loading requirements from a file.
>>> path_req = os.path.join(_PROJECT_ROOT, "requirements")
>>> load_requirements(path_req) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['numpy...', 'torch...', ...]
>>> load_requirements(path_req, unfreeze="major") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
['pytorch_lightning...', 'lightning_app...']
"""
with open(os.path.join(path_dir, file_name)) as file:
lines = [ln.strip() for ln in file.readlines()]
reqs = []
for ln in lines:
# filer all comments
comment = ""
if comment_char in ln:
comment = ln[ln.index(comment_char) :]
ln = ln[: ln.index(comment_char)]
req = ln.strip()
# skip directly installed dependencies
if not req or req.startswith("http") or "@http" in req:
continue
# remove version restrictions unless they are strict
if unfreeze and "<" in req and "strict" not in comment:
req = re.sub(r",? *<=? *[\d\.\*]+", "", req).strip()

# adding strict back to the comment
if "strict" in comment:
req += " # strict"

reqs.append(req)
return reqs
reqs.append(_augment_requirement(ln, comment_char=comment_char, unfreeze=unfreeze))
# filter empty lines
return [str(req) for req in reqs if req]


def load_readme_description(path_dir: str, homepage: str, version: str) -> str:
"""Load readme as decribtion.
>>> load_readme_description(_PROJECT_ROOT, "", "") # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
'<div align="center">...'
'...'
"""
path_readme = os.path.join(path_dir, "README.md")
text = open(path_readme, encoding="utf-8").read()
Expand Down Expand Up @@ -127,7 +177,18 @@ def replace_block_with_imports(lines: List[str], import_path: str, kword: str =
>>> lines = replace_block_with_imports(lines, import_path, "def")
"""
body, tracking, skip_offset = [], False, 0
for ln in lines:
for i, ln in enumerate(lines):
# support for defining a class with this condition
conditional_class_definitions = ("if TYPE_CHECKING", "if typing.TYPE_CHECKING", "if torch.", "if _TORCH_")
if (
any(ln.startswith(pattern) for pattern in conditional_class_definitions)
# avoid bug in CI for the <1.7 meta code
and "pytorch_lightning.utilities.meta" not in import_path
):
# dedent the next line
lines[i + 1] = lines[i + 1].lstrip()
continue

offset = len(ln) - len(ln.lstrip())
# in case of mating the class args are multi-line
if tracking and ln and offset <= skip_offset and not any(ln.lstrip().startswith(c) for c in ")]"):
Expand Down Expand Up @@ -439,12 +500,14 @@ def _download_frontend(root: str = _PROJECT_ROOT):
print("The Lightning UI downloading has failed!")


def _adjust_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
"""Parse the base requirements and append as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`."""
def _relax_require_versions(source_dir: str = "src", req_dir: str = "requirements") -> None:
"""Parse the base requirements and append as version adjustments if needed `pkg>=X1.Y1.Z1,==X2.Y2.*`.
>>> _relax_require_versions("../src", "../requirements")
"""
reqs = load_requirements(req_dir, file_name="base.txt")
for i, req in enumerate(reqs):
pkg_name = req[: min(req.index(c) for c in ">=" if c in req)]
ver_ = parse_version_from_file(os.path.join(source_dir, pkg_name))
for i, req in enumerate(parse_requirements(reqs)):
ver_ = parse_version_from_file(os.path.join(source_dir, req.name))
if not ver_:
continue
ver2 = ".".join(ver_.split(".")[:2] + ["*"])
Expand Down
46 changes: 35 additions & 11 deletions .azure/app-cloud-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,38 @@ trigger:
- "master"
- "release/*"
- "refs/tags/*"
paths:
include:
- ".azure/app-cloud-e2e.yml"
- "requirements/app/**"
- "src/lightning_app/**"
- "examples/app_*"

pr:
- "master"
- "release/*"
branches:
include:
- "master"
- "release/*"
paths:
include:
- ".azure/app-cloud-e2e.yml"
- "requirements/app/**"
- "src/lightning_app/**"
- "examples/app_*"

# variables are automatically exported as environment variables so this will override pip's default cache dir
variables:
# variables are automatically exported as environment variables so this will override pip's default cache dir
- name: pip_cache_dir
value: $(Pipeline.Workspace)/.pip
- name: local_id
value: $(Build.BuildId)

jobs:
- job: App_cloud_e2e_testing
pool: azure-cpus
container:
image: mcr.microsoft.com/playwright/python:v1.25.2-focal
options: "--shm-size=2g"
timeoutInMinutes: "30"
cancelTimeoutInMinutes: "2"
strategy:
matrix:
'App: v0_app':
Expand All @@ -54,20 +68,24 @@ jobs:
name: "payload"
'App: commands_and_api':
name: "commands_and_api"
timeoutInMinutes: "30"
cancelTimeoutInMinutes: "2"
# values: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#workspace
workspace:
clean: all
steps:

- script: echo '##vso[task.setvariable variable=local_id]$(System.PullRequest.PullRequestNumber)'
displayName: "Set id for this PR"
condition: eq(variables['Build.Reason'], 'PullRequest')

- bash: |
whoami
printf "local id: $(local_id)\n"
python --version
pip --version
displayName: 'Info'
# TODO: parse the PR number
- bash: |
ID=$(date +%s)
echo "##vso[task.setvariable variable=local_id]$ID"
- task: Cache@2
inputs:
key: 'pip | "$(name)" | requirements/app/base.txt'
Expand All @@ -90,12 +108,15 @@ jobs:
displayName: 'Install lightning'

- bash: |
rm -rf examples/app_template_jupyterlab || true
git clone https://github.com/Lightning-AI/LAI-lightning-template-jupyterlab-App examples/app_template_jupyterlab
cp examples/app_template_jupyterlab/tests/test_template_jupyterlab.py tests/tests_app_examples/test_template_jupyterlab.py
condition: eq(variables['name'], 'template_jupyterlab')
displayName: 'Clone Template Jupyter Lab Repo'
- bash: git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
- bash: |
rm -rf examples/app_template_react_ui || true
git clone https://github.com/Lightning-AI/lightning-template-react examples/app_template_react_ui
condition: eq(variables['name'], 'template_react_ui')
displayName: 'Clone Template React UI Repo'
Expand All @@ -119,14 +140,17 @@ jobs:
LIGHTNING_API_KEY: $(LIGHTNING_API_KEY_PROD)
LIGHTNING_USERNAME: $(LIGHTNING_USERNAME)
LIGHTNING_CLOUD_URL: $(LIGHTNING_CLOUD_URL_PROD)
LIGHTNING_DEBUG: '1'
displayName: 'Run the tests'
- publish: '$(Build.ArtifactStagingDirectory)/videos'
condition: failed()
displayName: 'Publish videos'
artifact: $(name)

- bash: |
time python -c "from lightning.app import testing; testing.delete_cloud_lightning_apps()"
condition: always()
env:
# LAI_USER: $(LAI_USER)
# LAI_PASS: $(LAI_PASS)
Expand Down
Loading

0 comments on commit 1040d35

Please sign in to comment.