Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

adapter compability messaging added. #4565

Merged
merged 5 commits into from
Feb 3, 2022
Merged

adapter compability messaging added. #4565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4689318
adapter compability messaging added.
nkyuray Jan 11, 2022
e4e8e97
edited plugin version compatibility message
nkyuray Jan 30, 2022
ff8ca6b
edited test version for plugin compability
nkyuray Jan 30, 2022
433f9fa
compare using only major and minor
ChenyuLInx Feb 3, 2022
260e2bb
Add checking PYPI and update changelog
ChenyuLInx Feb 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@
### Under the hood
- Testing cleanup ([#4496](https://github.com/dbt-labs/dbt-core/pull/4496), [#4509](https://github.com/dbt-labs/dbt-core/pull/4509))

## dbt-core 1.0.2 (TBD)
### Fixes
- adapter compability messaging added([#4438](https://github.com/dbt-labs/dbt-core/pull/4438) [#4565](https://github.com/dbt-labs/dbt-core/pull/4565))

Contributors:
* [@nkyuray](https://github.com/nkyuray) ([#4565](https://github.com/dbt-labs/dbt-core/pull/4565))
## dbt-core 1.0.1 (January 03, 2022)

## dbt-core 1.0.1rc1 (December 20, 2021)
Expand Down
48 changes: 40 additions & 8 deletions core/dbt/version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,15 @@
import dbt.exceptions
import dbt.semver

from dbt.ui import green, red, yellow
from dbt import flags

PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt-core/json'


def get_latest_version():
def get_latest_version(version_url: str = PYPI_VERSION_URL):
try:
resp = requests.get(PYPI_VERSION_URL)
resp = requests.get(version_url)
data = resp.json()
version_string = data['info']['version']
except (json.JSONDecodeError, KeyError, requests.RequestException):
Expand All @@ -29,7 +31,13 @@ def get_installed_version():
return dbt.semver.VersionSpecifier.from_version_string(__version__)


def get_package_pypi_url(package_name: str) -> str:
return f'https://pypi.org/pypi/dbt-{package_name}/json'


def get_version_information():
flags.USE_COLORS = True if not flags.USE_COLORS else None

installed = get_installed_version()
latest = get_latest_version()

Expand All @@ -44,16 +52,40 @@ def get_version_information():

plugin_version_msg = "Plugins:\n"
for plugin_name, version in _get_dbt_plugins_info():
plugin_version_msg += ' - {plugin_name}: {version}\n'.format(
plugin_name=plugin_name, version=version
)
plugin_version = dbt.semver.VersionSpecifier.from_version_string(version)
latest_plugin_version = get_latest_version(version_url=get_package_pypi_url(plugin_name))
plugin_update_msg = ''
if installed == plugin_version or (
latest_plugin_version and plugin_version == latest_plugin_version
):
Comment on lines +58 to +60
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's possible that plugins will put out patch releases before dbt-core puts out a patch release. Imagine:

  • dbt-core==1.0.1 and dbt-synapse==1.0.2 are both available
  • I have dbt-synapse==1.0.1 installed locally
  • installed == plugin_version will return True, at the same time, there's a compatible plugin update available

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't realize this case, and the rule in the next comment, I will open a new issue and get these changes in

compatibility_msg = green('Up to date!')
else:
if latest_plugin_version:
if installed.major == plugin_version.major:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

dbt-core + plugins are compatible at the minor-version level. So I think this might want to be:

Suggested change
if installed.major == plugin_version.major:
if installed.minor == plugin_version.minor:

If installed.minor != plugin_version.minor, we'd want to raise red('Not compatible!')

compatibility_msg = yellow('Update available!')
else:
compatibility_msg = red('Out of date!')
plugin_update_msg = (
" Your version of dbt-{} is out of date! "
"You can find instructions for upgrading here:\n"
" https://docs.getdbt.com/dbt-cli/install/overview\n\n"
).format(plugin_name)
else:
compatibility_msg = yellow('No PYPI version available')

plugin_version_msg += (
" - {}: {} - {}\n"
"{}"
).format(plugin_name, version, compatibility_msg, plugin_update_msg)

if latest is None:
return ("{}The latest version of dbt could not be determined!\n"
"Make sure that the following URL is accessible:\n{}\n\n{}"
.format(version_msg, PYPI_VERSION_URL, plugin_version_msg))
.format(version_msg, PYPI_VERSION_URL, plugin_version_msg)
)

if installed == latest:
return "{}Up to date!\n\n{}".format(version_msg, plugin_version_msg)
return f"{version_msg}{green('Up to date!')}\n\n{plugin_version_msg}"

elif installed > latest:
return ("{}Your version of dbt is ahead of the latest "
Expand Down Expand Up @@ -91,7 +123,7 @@ def _get_dbt_plugins_info():
f'dbt.adapters.{plugin_name}.__version__'
)
except ImportError:
# not an adpater
# not an adapter
continue
yield plugin_name, mod.version

Expand Down
131 changes: 92 additions & 39 deletions test/unit/test_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,45 @@

import dbt.main
import dbt.version
from dbt.ui import green, red, yellow


class MockResponse(object):

def __init__(self, content: dict) -> None:
self.content = content

def json(self) -> dict:
return self.content

class VersionTest(unittest.TestCase):

@patch("dbt.version.__version__", "0.10.0")
@patch('dbt.version._get_dbt_plugins_info', autospec=True)
@patch('dbt.version.requests.get')
def test_versions_equal(self, mock_get, mock_get_dbt_plugins_info):
mock_get.return_value.json.return_value = {
'info': {'version': '0.10.0'}
'info': {'version': '0.10.0'},
}
mock_get_dbt_plugins_info.return_value = [
('dbt-postgres', '0.10.0'),
('dbt-redshift', '0.10.0'),
('dbt-bigquery', '0.10.0'),
('dbt-snowflake', '0.10.0')
('postgres', '0.10.0'),
('redshift', '0.10.0'),
('bigquery', '0.10.0'),
('snowflake', '0.10.0')
]

latest_version = dbt.version.get_latest_version()
installed_version = dbt.version.get_installed_version()
version_information = dbt.version.get_version_information()

expected_version_information = "installed version: 0.10.0\n" \
" latest version: 0.10.0\n\n" \
"Up to date!\n\n" \
"Plugins:\n" \
" - dbt-postgres: 0.10.0\n" \
" - dbt-redshift: 0.10.0\n" \
" - dbt-bigquery: 0.10.0\n" \
" - dbt-snowflake: 0.10.0\n"

" latest version: 0.10.0\n\n" \
f"{green('Up to date!')}\n\n" \
"Plugins:\n" \
f" - postgres: 0.10.0 - {green('Up to date!')}\n" \
f" - redshift: 0.10.0 - {green('Up to date!')}\n" \
f" - bigquery: 0.10.0 - {green('Up to date!')}\n" \
f" - snowflake: 0.10.0 - {green('Up to date!')}\n"
self.assertEqual(latest_version, installed_version)
self.assertEqual(latest_version, installed_version)
self.assertMultiLineEqual(version_information,
Expand All @@ -47,23 +55,23 @@ def test_installed_version_greater(self, mock_get, mock_get_dbt_plugins_info):
'info': {'version': '0.10.1'}
}
mock_get_dbt_plugins_info.return_value = [
('dbt-postgres', '0.10.0'),
('dbt-redshift', '0.10.0'),
('dbt-bigquery', '0.10.0'),
('dbt-snowflake', '0.10.0')
('postgres', '0.10.2-a1'),
('redshift', '0.10.2-a1'),
('bigquery', '0.10.2-a1'),
('snowflake', '0.10.2-a1')
]
latest_version = dbt.version.get_latest_version()
installed_version = dbt.version.get_installed_version()
version_information = dbt.version.get_version_information()

expected_version_information = "installed version: 0.10.2-a1\n" \
" latest version: 0.10.1\n\n" \
"Your version of dbt is ahead of the latest release!\n\n" \
"Plugins:\n" \
" - dbt-postgres: 0.10.0\n" \
" - dbt-redshift: 0.10.0\n" \
" - dbt-bigquery: 0.10.0\n" \
" - dbt-snowflake: 0.10.0\n"
" latest version: 0.10.1\n\n" \
"Your version of dbt is ahead of the latest release!\n\n" \
"Plugins:\n" \
f" - postgres: 0.10.2-a1 - {green('Up to date!')}\n" \
f" - redshift: 0.10.2-a1 - {green('Up to date!')}\n" \
f" - bigquery: 0.10.2-a1 - {green('Up to date!')}\n" \
f" - snowflake: 0.10.2-a1 - {green('Up to date!')}\n"

assert installed_version > latest_version
self.assertMultiLineEqual(version_information,
Expand All @@ -73,34 +81,77 @@ def test_installed_version_greater(self, mock_get, mock_get_dbt_plugins_info):
@patch('dbt.version._get_dbt_plugins_info', autospec=True)
@patch('dbt.version.requests.get')
def test_installed_version_lower(self, mock_get, mock_get_dbt_plugins_info):

mock_get.return_value.json.return_value = {
'info': {'version': '0.10.0'}
}
mock_get_dbt_plugins_info.return_value = [
('dbt-postgres', '0.10.0'),
('dbt-redshift', '0.10.0'),
('dbt-bigquery', '0.10.0'),
('dbt-snowflake', '0.10.0')
('postgres', '0.9.5'),
('redshift', '0.9.5'),
('bigquery', '0.9.5'),
('snowflake', '0.9.5')
]
latest_version = dbt.version.get_latest_version()
installed_version = dbt.version.get_installed_version()
version_information = dbt.version.get_version_information()

expected_version_information = "installed version: 0.9.5\n" \
" latest version: 0.10.0\n\n" \
"Your version of dbt is out of date! " \
"You can find instructions for upgrading here:\n" \
"https://docs.getdbt.com/docs/installation\n\n" \
"Plugins:\n" \
" - dbt-postgres: 0.10.0\n" \
" - dbt-redshift: 0.10.0\n" \
" - dbt-bigquery: 0.10.0\n" \
" - dbt-snowflake: 0.10.0\n"

" latest version: 0.10.0\n\n" \
"Your version of dbt is out of date! " \
"You can find instructions for upgrading here:\n" \
"https://docs.getdbt.com/docs/installation\n\n" \
"Plugins:\n" \
f" - postgres: 0.9.5 - {green('Up to date!')}\n" \
f" - redshift: 0.9.5 - {green('Up to date!')}\n" \
f" - bigquery: 0.9.5 - {green('Up to date!')}\n" \
f" - snowflake: 0.9.5 - {green('Up to date!')}\n"
assert installed_version < latest_version
self.assertMultiLineEqual(version_information,
expected_version_information)

@patch("dbt.version.__version__", "1.0.1")
@patch('dbt.version._get_dbt_plugins_info', autospec=True)
@patch('dbt.version.requests.get')
def test_installed_version_major_minor_pypi(self, mock_get, mock_get_dbt_plugins_info):
mock_get.side_effect = [
MockResponse({'info': {'version': '1.0.1'}}), # version for dbt-core
MockResponse({'info': {'version': '1.0.1'}}), # version for dbt-postgres
MockResponse({'info': {'version': '1.0.1'}}), # version for dbt-redshift
MockResponse({'info': {'version': '1.0.0'}}), # version for dbt-bigquery
MockResponse({'info': {'version': '1.0.1'}}), # version for dbt-snowflake
KeyError('no PYPI Version'), # no PYPI registry for newdb1
KeyError('no PYPI Version') # no PYPI registry for newdb2
]

mock_get_dbt_plugins_info.return_value = [
('postgres', '1.0.1'),
('redshift', '1.0.0'),
('bigquery', '1.0.0'),
('snowflake', '0.10.0'),
('newdb1', '1.0.1'),
('newdb2', '0.5.1'),
]
version_information = dbt.version.get_version_information()
expected_version_information = "installed version: 1.0.1\n" \
" latest version: 1.0.1\n\n" \
f"{green('Up to date!')}\n\n" \
"Plugins:\n" \
f" - postgres: 1.0.1 - {green('Up to date!')}\n" \
f" - redshift: 1.0.0 - {yellow('Update available!')}\n" \
" Your version of dbt-redshift is out of date! You can find instructions for " \
"upgrading here:\n" \
" https://docs.getdbt.com/dbt-cli/install/overview\n\n" \
f" - bigquery: 1.0.0 - {green('Up to date!')}\n" \
f" - snowflake: 0.10.0 - {red('Out of date!')}\n" \
" Your version of dbt-snowflake is out of date! You can find instructions for " \
"upgrading here:\n" \
" https://docs.getdbt.com/dbt-cli/install/overview\n\n"\
f" - newdb1: 1.0.1 - {green('Up to date!')}\n" \
f" - newdb2: 0.5.1 - {yellow('No PYPI version available')}\n"

self.assertMultiLineEqual(version_information,
expected_version_information)

# suppress having version info printed to the screen during tests.
@patch('sys.stderr')
@patch('dbt.version.requests.get')
Expand All @@ -125,6 +176,7 @@ def glob_side_effect(path: str) -> str:
path.replace('*', 'postgres'),
path.replace('*', 'snowflake'),
]

mock_glob.side_effect = glob_side_effect
self.assertEqual(
list(dbt.version._get_adapter_plugin_names()),
Expand All @@ -148,6 +200,7 @@ def glob_side_effect(path: str) -> str:
return [path.replace('*', 'postgres')]
elif 'snowflake' in path:
return [path.replace('*', 'snowflake')]

mock_glob.side_effect = glob_side_effect
self.assertEqual(
list(dbt.version._get_adapter_plugin_names()),
Expand All @@ -159,7 +212,7 @@ def glob_side_effect(path: str) -> str:
@patch('dbt.version._get_adapter_plugin_names', autospec=True)
@patch('importlib.import_module', autospec=True)
def test_get_dbt_plugins_info_with_version_info(
self, mock_mod, mock_get_plugin_names
self, mock_mod, mock_get_plugin_names
):
mock_get_plugin_names.return_value = ['postgres', 'snowflake']
mod_version = unittest.mock.Mock()
Expand Down