ci: bump ubuntu version, remove mockldap (#2013)

* ci: fix removed ubuntu, bump version * remove python-dev * try1 * bump python * bump python * try2 * try3 * try4 * bumps * add openldap test integration * fix requirements1 * fix ci * bump mongoengine * bump mongoengine * test CI * test CI * test CI * test CI * test CI * test CI * test CI * test CI * fix mssql * fix mongodb deps * fix mongodb deps * fix mongodb deps * fix mongodb deps * fix mongodb deps * add 3.7 back in
dpgaspar · Apr 20, 2023 · 0e08270 · 0e08270
1 parent c7477aa
commit 0e08270
Show file tree

Hide file tree

Showing 11 changed files with 389 additions and 508 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,21 +9,21 @@ on:
 
 jobs:
   lint:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: [3.8]
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
+          sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev
           pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
@@ -35,10 +35,10 @@ jobs:
         run: mypy flask_appbuilder
 
   test-postgres:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9.7]
+        python-version: ["3.7", "3.8", "3.9"]
     env:
       SQLALCHEMY_DATABASE_URI:
         postgresql+psycopg2://pguser:[email protected]:15432/app
@@ -52,15 +52,27 @@ jobs:
         ports:
           - 15432:5432
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Run openldap
+        run: |
+          docker run -d \
+            -v '${{ github.workspace }}/docker/openldap/ldifs:/ldifs' \
+            -v '${{ github.workspace }}/docker/openldap/schemas/memberof.ldif:/opt/bitnami/openldap/etc/schema/memberof.ldif' \
+            -e LDAP_URI=ldap://openldap:1389 \
+            -e LDAP_BASE=dc=example,dc=org \
+            -e LDAP_ADMIN_USERNAME=admin \
+            -e LDAP_ADMIN_PASSWORD=admin_password \
+            -e LDAP_EXTRA_SCHEMAS=cosine,inetorgperson,nis,memberof \
+            -p 1389:1389 \
+            bitnami/openldap:2.6.4
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
+          sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev
           pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
@@ -73,10 +85,10 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -cF python
 
   test-mysql:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       SQLALCHEMY_DATABASE_URI: |
         mysql+mysqldb://mysqluser:[email protected]:13306/app?charset=utf8mb4&binary_prefix=true
@@ -91,15 +103,27 @@ jobs:
         ports:
           - 13306:3306
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Run openldap
+        run: |
+          docker run -d \
+            -v '${{ github.workspace }}/docker/openldap/ldifs:/ldifs' \
+            -v '${{ github.workspace }}/docker/openldap/schemas/memberof.ldif:/opt/bitnami/openldap/etc/schema/memberof.ldif' \
+            -e LDAP_URI=ldap://openldap:1389 \
+            -e LDAP_BASE=dc=example,dc=org \
+            -e LDAP_ADMIN_USERNAME=admin \
+            -e LDAP_ADMIN_PASSWORD=admin_password \
+            -e LDAP_EXTRA_SCHEMAS=cosine,inetorgperson,nis,memberof \
+            -p 1389:1389 \
+            bitnami/openldap:2.6.4
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
+          sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev
           pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
@@ -112,10 +136,10 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -cF python
 
   test-mssql:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.7]
+        python-version: [3.8]
     env:
       SQLALCHEMY_DATABASE_URI: |
         mssql+pyodbc://sa:Password_123@localhost:11433/master?driver=FreeTDS
@@ -128,15 +152,27 @@ jobs:
         ports:
           - 11433:1433
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Run openldap
+        run: |
+          docker run -d \
+            -v '${{ github.workspace }}/docker/openldap/ldifs:/ldifs' \
+            -v '${{ github.workspace }}/docker/openldap/schemas/memberof.ldif:/opt/bitnami/openldap/etc/schema/memberof.ldif' \
+            -e LDAP_URI=ldap://openldap:1389 \
+            -e LDAP_BASE=dc=example,dc=org \
+            -e LDAP_ADMIN_USERNAME=admin \
+            -e LDAP_ADMIN_PASSWORD=admin_password \
+            -e LDAP_EXTRA_SCHEMAS=cosine,inetorgperson,nis,memberof \
+            -p 1389:1389 \
+            bitnami/openldap:2.6.4
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev freetds-bin unixodbc-dev tdsodbc
+          sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev freetds-bin unixodbc-dev tdsodbc
           pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
@@ -150,29 +186,42 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -cF python
 
   test-mongodb:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: [3.7]
     services:
-      mssql:
+      mongo:
         image: mongo:4.4.1-bionic
         ports:
           - 27017:27017
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Run openldap
+        run: |
+          docker run -d \
+            -v '${{ github.workspace }}/docker/openldap/ldifs:/ldifs' \
+            -v '${{ github.workspace }}/docker/openldap/schemas/memberof.ldif:/opt/bitnami/openldap/etc/schema/memberof.ldif' \
+            -e LDAP_URI=ldap://openldap:1389 \
+            -e LDAP_BASE=dc=example,dc=org \
+            -e LDAP_ADMIN_USERNAME=admin \
+            -e LDAP_ADMIN_PASSWORD=admin_password \
+            -e LDAP_EXTRA_SCHEMAS=cosine,inetorgperson,nis,memberof \
+            -p 1389:1389 \
+            bitnami/openldap:2.6.4
       - name: Install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev
+          sudo apt-get install -y libldap2-dev libsasl2-dev libssl-dev
           pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
           pip install -r requirements-extra.txt
+          pip install -r requirements-mongodb.txt
       - name: Run tests
         run: |
           nosetests --stop -v --with-coverage --cover-package=flask_appbuilder flask_appbuilder/tests/test_mongoengine.py

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,3 +20,18 @@ services:
       MONGO_INITDB_DATABASE: app
     ports:
       - 27017:27017
+  ldap:
+    container_name: fab-ldap
+    image: bitnami/openldap:2.6.4
+    environment:
+      LDAP_URI: ldap://openldap:1389
+      LDAP_BASE: dc=example,dc=org
+      LDAP_ADMIN_USERNAME: admin
+      LDAP_ADMIN_PASSWORD: admin_password
+      LDAP_CUSTOM_LDIF_DIR: /ldifs
+      LDAP_EXTRA_SCHEMAS: cosine,inetorgperson,nis,memberof
+    volumes:
+      - './docker/openldap/ldifs:/ldifs'
+      - './docker/openldap/schemas/memberof.ldif:/opt/bitnami/openldap/etc/schema/memberof.ldif'
+    ports:
+      - 1389:1389
diff --git a/docker/openldap/ldifs/users.ldif b/docker/openldap/ldifs/users.ldif
@@ -0,0 +1,68 @@
+# extended LDIF
+#
+# LDAPv3
+# base <dc=example,dc=org> with scope subtree
+# filter: (objectclass=*)
+# requesting: ALL
+#
+
+# example.org
+dn: dc=example,dc=org
+objectClass: dcObject
+objectClass: organization
+dc: example
+o: example
+
+# users, example.org
+dn: ou=users,dc=example,dc=org
+objectClass: organizationalUnit
+ou: users
+
+# users, example.org
+dn: ou=groups,dc=example,dc=org
+objectClass: organizationalUnit
+ou: groups
+
+# alice, users, example.org
+dn: cn=alice,ou=users,dc=example,dc=org
+cn: alice
+sn: Doe
+givenName: Alice
+objectClass: inetOrgPerson
+objectClass: posixAccount
+objectClass: shadowAccount
+userPassword: alice_password
+uid: alice
+uidNumber: 1000
+gidNumber: 1000
+mail: [email protected]
+homeDirectory: /home/alice
+
+# natalie, users, example.org
+dn: cn=natalie,ou=users,dc=example,dc=org
+cn: natalie
+sn: Smith
+givenName: Natalie
+objectClass: inetOrgPerson
+objectClass: posixAccount
+objectClass: shadowAccount
+userPassword: natalie_password
+uid: natalie
+uidNumber: 1001
+gidNumber: 1001
+mail: [email protected]
+homeDirectory: /home/natalie
+
+# readers, users, example.org
+dn: cn=readers,ou=groups,dc=example,dc=org
+cn: readers
+objectClass: groupOfNames
+objectClass: top
+member: cn=alice,ou=users,dc=example,dc=org
+member: cn=natalie,ou=users,dc=example,dc=org
+
+dn: cn=staff,ou=groups,dc=example,dc=org
+cn: staff
+objectClass: groupOfNames
+objectClass: top
+member: cn=alice,ou=users,dc=example,dc=org
diff --git a/docker/openldap/schemas/memberof.ldif b/docker/openldap/schemas/memberof.ldif
@@ -0,0 +1,19 @@
+dn: cn=module,cn=config
+cn: module
+objectClass: olcModuleList
+olcModulePath: /opt/bitnami/openldap/lib/openldap
+olcModuleLoad: memberof.so
+olcModuleLoad: refint.so
+
+dn: olcOverlay=memberof,olcDatabase={2}mdb,cn=config
+objectClass: olcMemberOf
+objectClass: olcOverlayConfig
+olcOverlay: memberof
+
+dn: olcOverlay=refint,olcDatabase={2}mdb,cn=config
+objectClass: olcConfig
+objectClass: olcOverlayConfig
+objectClass: olcRefintConfig
+objectClass: top
+olcOverlay: refint
+olcRefintAttribute: memberof member manager owner
diff --git a/flask_appbuilder/models/sqla/filters.py b/flask_appbuilder/models/sqla/filters.py
@@ -34,13 +34,13 @@
 
 def get_field_setup_query(query, model, column_name):
     """
-        Help function for SQLA filters, checks for dot notation on column names.
-        If it exists, will join the query with the model
-        from the first part of the field name.
+    Help function for SQLA filters, checks for dot notation on column names.
+    If it exists, will join the query with the model
+    from the first part of the field name.
 
-        example:
-            Contact.created_by: if created_by is a User model,
-            it will be joined to the query.
+    example:
+        Contact.created_by: if created_by is a User model,
+        it will be joined to the query.
     """
     if not hasattr(model, column_name):
         # it's an inner obj attr
@@ -195,7 +195,11 @@ def apply(self, query, value):
             logging.warning(
                 "Filter exception for %s with value %s, will not apply", field, value
             )
-            self.datamodel.session.rollback()
+            try:
+                self.datamodel.session.rollback()
+            except SQLAlchemyError:
+                # on MSSQL a rollback would fail here
+                pass
             raise ApplyFilterException(exception=exc)
         return query.filter(field == rel_obj)
 
@@ -212,7 +216,11 @@ def apply(self, query, value):
             logging.warning(
                 "Filter exception for %s with value %s, will not apply", field, value
             )
-            self.datamodel.session.rollback()
+            try:
+                self.datamodel.session.rollback()
+            except SQLAlchemyError:
+                # on MSSQL a rollback would fail here
+                pass
             raise ApplyFilterException(exception=exc)
         return query.filter(field != rel_obj)
 
@@ -234,7 +242,11 @@ def apply_item(self, query, field, value_item):
                 field,
                 value_item,
             )
-            self.datamodel.session.rollback()
+            try:
+                self.datamodel.session.rollback()
+            except SQLAlchemyError:
+                # on MSSQL a rollback would fail here
+                pass
             raise ApplyFilterException(exception=exc)
 
         if rel_obj:
@@ -279,8 +291,8 @@ def apply(self, query, func):
 
 class SQLAFilterConverter(BaseFilterConverter):
     """
-        Class for converting columns into a supported list of filters
-        specific for SQLAlchemy.
+    Class for converting columns into a supported list of filters
+    specific for SQLAlchemy.
 
     """