Merge branch 'master' of github.com:apache/superset into custom-theme

� Conflicts:
�	superset-frontend/src/preamble.ts

RELEASING/README.md

@@ -118,6 +118,8 @@ git push upstream $SUPERSET_GITHUB_BRANCH
 Next, update the `CHANGELOG.md` with all the changes that are included in the release.
 Make sure the branch has been pushed to `upstream` to ensure the changelog generator
 can pick up changes since the previous release.
+Change log script requires a github token and will try to use your env var GITHUB_TOKEN.
+you can also pass the token using the parameter `--access_token`.
 
 Example:
 ```bash

RELEASING/changelog.py

@@ -17,18 +17,22 @@
 # pylint: disable=no-value-for-parameter
 
 import csv as lib_csv
-import json
 import os
 import re
 import sys
 from dataclasses import dataclass
-from time import sleep
 from typing import Any, Dict, Iterator, List, Optional, Union
-from urllib import request
-from urllib.error import HTTPError
 
 import click
 
+try:
+    from github import BadCredentialsException, Github, PullRequest
+except ModuleNotFoundError:
+    print("PyGithub is a required package for this script")
+    exit(1)
+
+SUPERSET_REPO = "apache/superset"
+
 
 @dataclass
 class GitLog:
@@ -60,47 +64,31 @@ class GitChangeLog:
     We want to map a git author to a github login, for that we call github's API
     """
 
-    def __init__(self, version: str, logs: List[GitLog]) -> None:
+    def __init__(
+        self, version: str, logs: List[GitLog], access_token: Optional[str] = None
+    ) -> None:
         self._version = version
         self._logs = logs
         self._github_login_cache: Dict[str, Optional[str]] = {}
         self._wait = 10
+        github_token = access_token or os.environ.get("GITHUB_TOKEN")
+        self._github = Github(github_token)
+        self._superset_repo = ""
 
-    def _wait_github_rate_limit(self) -> None:
-        """
-        Waits for available rate limit slots on the github API
-        """
-        while True:
-            rate_limit_payload = self._fetch_github_rate_limit()
-            if rate_limit_payload["rate"]["remaining"] > 1:
-                break
-            print(".", end="", flush=True)
-            sleep(self._wait)
-        print()
-
-    @staticmethod
-    def _fetch_github_rate_limit() -> Dict[str, Any]:
-        """
-        Fetches current github rate limit info
-        """
-        with request.urlopen("https://api.github.com/rate_limit") as response:
-            payload = json.loads(response.read())
-        return payload
-
-    def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
+    def _fetch_github_pr(self, pr_number: int) -> PullRequest:
         """
         Fetches a github PR info
         """
-        payload = {}
         try:
-            self._wait_github_rate_limit()
-            with request.urlopen(
-                "https://api.github.com/repos/apache/superset/pulls/" f"{pr_number}"
-            ) as response:
-                payload = json.loads(response.read())
-        except HTTPError as ex:
-            print(f"{ex}", flush=True)
-        return payload
+            github_repo = self._github.get_repo(SUPERSET_REPO)
+        except BadCredentialsException as ex:
+            print(
+                f"Bad credentials to github provided"
+                f" use access_token parameter or set GITHUB_TOKEN"
+            )
+            sys.exit(1)
+
+        return github_repo.get_pull(pr_number)
 
     def _get_github_login(self, git_log: GitLog) -> Optional[str]:
         """
@@ -113,7 +101,7 @@ def _get_github_login(self, git_log: GitLog) -> Optional[str]:
         if git_log.pr_number:
             pr_info = self._fetch_github_pr(git_log.pr_number)
             if pr_info:
-                github_login = pr_info["user"]["login"]
+                github_login = pr_info.user.login
             else:
                 github_login = author_name
         # set cache
@@ -131,7 +119,7 @@ def __repr__(self) -> str:
                 github_login = log.author
             result = result + (
                 f"- [#{log.pr_number}]"
-                f"(https://github.com/apache/superset/pull/{log.pr_number}) "
+                f"(https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}) "
                 f"{log.message} (@{github_login})\n"
             )
             print(f"\r {i}/{len(self._logs)}", end="", flush=True)
@@ -141,7 +129,7 @@ def __iter__(self) -> Iterator[Dict[str, Any]]:
         for log in self._logs:
             yield {
                 "pr_number": log.pr_number,
-                "pr_link": f"https://github.com/apache/superset/pull/"
+                "pr_link": f"https://github.com/{SUPERSET_REPO}/pull/"
                 f"{log.pr_number}",
                 "message": log.message,
                 "time": log.time,
@@ -276,13 +264,20 @@ def compare(base_parameters: BaseParameters) -> None:
 @click.option(
     "--csv", help="The csv filename to export the changelog to",
 )
+@click.option(
+    "--access_token",
+    help="The github access token,"
+    " if not provided will try to fetch from GITHUB_TOKEN env var",
+)
 @click.pass_obj
-def change_log(base_parameters: BaseParameters, csv: str) -> None:
+def change_log(base_parameters: BaseParameters, csv: str, access_token: str) -> None:
     """ Outputs a changelog (by PR) """
     previous_logs = base_parameters.previous_logs
     current_logs = base_parameters.current_logs
     previous_diff_logs = previous_logs.diff(current_logs)
-    logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
+    logs = GitChangeLog(
+        current_logs.git_ref, previous_diff_logs[::-1], access_token=access_token
+    )
     if csv:
         with open(csv, "w") as csv_file:
             log_items = list(logs)

helm/superset/templates/configmap-superset.yaml

@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.extraConfigs }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "superset.fullname" . }}-extra-config
+  labels:
+    app: {{ template "superset.name" . }}
+    chart: {{ template "superset.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+{{- range $path, $config := .Values.extraConfigs }}
+  {{ $path }}: |
+{{ tpl $config . | indent 4 -}}
+{{- end -}}
+{{- end -}}

helm/superset/templates/init-job.yaml

@@ -51,10 +51,20 @@ spec:
           - name: superset-config
             mountPath: {{ .Values.configMountPath | quote }}
             readOnly: true
+        {{ if .Values.extraConfigs }}
+          - name: superset-extra-config
+            mountPath: {{ .Values.extraConfigMountPath | quote }}
+            readOnly: true
+        {{- end }}
         command: {{  tpl (toJson .Values.init.command) . }}
       volumes:
         - name: superset-config
           secret:
             secretName: {{ tpl .Values.configFromSecret . }}
+        {{- if .Values.extraConfigs }}
+        - name: superset-extra-config
+          configMap:
+            name: {{ template "superset.fullname" . }}-extra-config
+        {{- end }}
       restartPolicy: Never
 {{- end }}

helm/superset/values.yaml

@@ -41,8 +41,23 @@ envFromSecret: '{{ template "superset.fullname" . }}-env'
 ##
 extraEnv: {}
 
+extraConfigs: {}
+  # datasources-init.yaml: | 
+  #     databases:
+  #     - allow_csv_upload: true
+  #       allow_ctas: true
+  #       allow_cvas: true
+  #       database_name: example-db
+  #       extra: "{\r\n    \"metadata_params\": {},\r\n    \"engine_params\": {},\r\n    \"\
+  #         metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_csv_upload\": []\r\n\
+  #         }"
+  #       sqlalchemy_uri: example://example-db.local
+  #       tables: []
+
 configMountPath: "/app/pythonpath"
 
+extraConfigMountPath: "/app/configs"
+
 image:
   repository: apache/superset
   tag: latest
@@ -256,4 +271,4 @@ nodeSelector: {}
 
 tolerations: []
 
-affinity: {}
+affinity: {}

requirements/base.txt

@@ -77,7 +77,7 @@ prison==0.1.3             # via flask-appbuilder
 py==1.9.0                 # via retry
 pyarrow==1.0.1            # via apache-superset
 pycparser==2.20           # via cffi
-pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
+pyjwt==1.7.1              # via apache-superset, flask-appbuilder, flask-jwt-extended
 pymeeus==0.3.7            # via convertdate
 pyparsing==2.4.7          # via apache-superset, packaging
 pyrsistent==0.16.1        # via -r requirements/base.in, jsonschema
@@ -92,12 +92,12 @@ redis==3.5.3              # via apache-superset
 retry==0.9.2              # via apache-superset
 selenium==3.141.0         # via apache-superset
 simplejson==3.17.2        # via apache-superset
-six==1.15.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
+six==1.15.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
 slackclient==2.5.0        # via apache-superset
 sqlalchemy-utils==0.36.8  # via apache-superset, flask-appbuilder
 sqlalchemy==1.3.20        # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 sqlparse==0.3.0           # via apache-superset
-typing-extensions==3.7.4.3  # via aiohttp, yarl
+typing-extensions==3.7.4.3  # via aiohttp, apache-superset, yarl
 urllib3==1.25.11          # via selenium
 vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach

requirements/development.in

@@ -24,3 +24,4 @@ pyhive[hive]>=0.6.1
 psycopg2-binary==2.8.5
 tableschema
 thrift>=0.11.0,<1.0.0
+pygithub>=1.54.1,<2.0.0

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:ae0364cae066a5cb8fb543c4f568bfcdacb6c1b7
+# SHA1:b7181d683bed10ffe4892d2f07bc94a503f46b44
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -11,6 +11,7 @@ boto3==1.16.10            # via tabulator
 botocore==1.19.10         # via boto3, s3transfer
 cached-property==1.5.2    # via tableschema
 certifi==2020.6.20        # via requests
+deprecated==1.2.11        # via pygithub
 et-xmlfile==1.0.1         # via openpyxl
 flask-cors==3.0.9         # via -r requirements/development.in
 future==0.18.2            # via pyhive
@@ -24,8 +25,9 @@ openpyxl==3.0.5           # via tabulator
 pillow==7.2.0             # via -r requirements/development.in
 psycopg2-binary==2.8.5    # via -r requirements/development.in
 pydruid==0.6.1            # via -r requirements/development.in
+pygithub==1.54.1          # via -r requirements/development.in
 pyhive[hive]==0.6.3       # via -r requirements/development.in
-requests==2.24.0          # via pydruid, tableschema, tabulator
+requests==2.24.0          # via pydruid, pygithub, tableschema, tabulator
 rfc3986==1.4.0            # via tableschema
 s3transfer==0.3.3         # via boto3
 sasl==0.2.1               # via pyhive, thrift-sasl
@@ -34,6 +36,7 @@ tabulator==1.52.5         # via tableschema
 thrift-sasl==0.4.2        # via pyhive
 thrift==0.13.0            # via -r requirements/development.in, pyhive, thrift-sasl
 unicodecsv==0.14.1        # via tableschema, tabulator
+wrapt==1.12.1             # via deprecated
 xlrd==1.2.0               # via tabulator
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/testing.txt

@@ -40,7 +40,6 @@ traitlets==5.0.5          # via ipython
 typed-ast==1.4.1          # via astroid
 wcwidth==0.2.5            # via prompt-toolkit
 websocket-client==0.57.0  # via docker
-wrapt==1.12.1             # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

superset-frontend/cypress-base/cypress/integration/explore/control.test.ts

@@ -104,7 +104,7 @@ describe('VizType control', () => {
       numScripts = nodes.length;
     });
 
-    cy.get('.Control .label').contains('Table').click();
+    cy.get('[data-test="visualization-type"]').contains('Table').click();
 
     cy.get('[role="button"]').contains('Line Chart').click();
 

superset-frontend/cypress-base/cypress/integration/explore/visualizations/table.test.ts

@@ -161,7 +161,7 @@ describe('Visualization > Table', () => {
       cy.verifySliceContainer('table');
       expect(response?.body.result[0].data.length).to.eq(limit);
     });
-    cy.get('span.label-danger').contains('10 rows');
+    cy.get('[data-test="row-count-label"]').contains('10 rows');
   });
 
   it('Test table with columns and row limit', () => {