Python 3.12

.github/workflows/test.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
     services:
       postgres:
         image: postgres
@@ -34,7 +34,7 @@ jobs:
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
           if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi
-          pip install pylint==2.12.2 pytest==7.1.2 codecov
+          pip install 'pylint==2.12.2 ; python_version < "3.11"' 'pylint==3.0.2 ; python_version >= "3.11"' pytest==7.1.2 codecov
       - name: Lint with pylint
         run: |
           pylint --rcfile=.pylintrc olo

olo/compat.py

@@ -11,6 +11,7 @@
 PY2 = sys.version_info[0] == 2
 PY39 = sys.version_info[:2] >= (3, 9)
 PY310 = sys.version_info[:2] >= (3, 10)
+PY311 = sys.version_info[:2] >= (3, 11)
 
 if PY2:
     from future_builtins import zip as izip, map as imap

olo/libs/decompiling.py

@@ -6,7 +6,7 @@
 # and is released under the Apache 2.0 license: https://www.apache.org/licenses/LICENSE-2.0
 
 from __future__ import absolute_import, print_function, division
-from olo.compat import PY2, PY310, izip, xrange
+from olo.compat import PY2, PY310, PY311, izip, xrange
 
 import sys
 import types
@@ -137,18 +137,30 @@ def decompile(decompiler):
                                 oparg * 65536
                             )
                             i += 2
+
+                opname = opnames[op].replace('+', '_')
                 if op >= HAVE_ARGUMENT:
                     if op in hasconst:
                         arg = [code.co_consts[oparg]]
                     elif op in hasname:
-                        arg = [code.co_names[oparg]]
+                        if opname == 'LOAD_GLOBAL':
+                            push_null = False
+                            if PY311:
+                                push_null = oparg & 1
+                                oparg >>= 1
+                            arg = [code.co_names[oparg], push_null]
+                        else:
+                            arg = [code.co_names[oparg]]
                     elif op in hasjrel:
-                        arg = [i + oparg * (2 if PY310 else 1)]
+                        arg = [i + oparg * (2 if PY310 else 1)
+                               * (-1 if 'BACKWARD' in opname else 1)]
                     elif op in haslocal:
                         arg = [code.co_varnames[oparg]]
                     elif op in hascompare:
                         arg = [cmp_op[oparg]]
                     elif op in hasfree:
+                        if PY311:
+                            oparg -= len(code.co_varnames)
                         arg = [free[oparg]]
                     elif op in hasjabs:
                         arg = [oparg * (2 if PY310 else 1)]
@@ -317,11 +329,38 @@ def CALL_METHOD(decompiler, argc):
         method = pop()
         return ast.CallFunc(method, args)
 
+    def CACHE(decompiler):
+        pass
+
+    def CALL(decompiler, argc):
+        values = decompiler.pop_items(argc)
+
+        keys = decompiler.kw_names
+        decompiler.kw_names = None
+
+        args = values
+        keywords = []
+        if keys:
+            args = values[:-len(keys)]
+            keywords = [ast.keyword(k, v) for k, v in zip(keys, values[-len(keys):])]
+
+        self = decompiler.stack.pop()
+        callable_ = decompiler.stack.pop()
+        if callable_ is None:
+            callable_ = self
+        else:
+            args.insert(0, self)
+        decompiler.stack.append(callable_)
+        return decompiler._call_function(args, keywords)
+
     def COMPARE_OP(decompiler, op):
         oper2 = decompiler.stack.pop()
         oper1 = decompiler.stack.pop()
         return ast.Compare(oper1, [(op, oper2)])
 
+    def COPY_FREE_VARS(decompiler, n):
+        pass
+
     def DUP_TOP(decompiler):
         return decompiler.stack[-1]
 
@@ -405,9 +444,37 @@ def JUMP_FORWARD(decompiler, endpos):
             decompiler.targets[endpos] = if_exp
         return if_exp
 
-    def LIST_APPEND(decompiler, offset=None):
-        throw(InvalidQuery, 'Use generator expression (... for ... in ...) '
-              'instead of list comprehension [... for ... in ...] inside query')  # noqa
+
+    def LIST_APPEND(decompiler, offset):
+        tos = decompiler.stack.pop()
+        list_node = decompiler.stack[-offset]
+        if isinstance(list_node, ast.comprehension):
+            throw(InvalidQuery('Use generator expression (... for ... in ...) '
+                               'instead of list comprehension [... for ... in ...] inside query'))
+
+        assert isinstance(list_node, ast.List), list_node
+        list_node.elts.append(tos)
+
+    def LIST_EXTEND(decompiler, offset):
+        if offset != 1:
+            raise NotImplementedError(offset)
+        items = decompiler.stack.pop()
+        if not isinstance(items, ast.Constant):
+            raise NotImplementedError(type(items))
+        if not isinstance(items.value, tuple):
+            raise NotImplementedError(type(items.value))
+        lst = decompiler.stack.pop()
+        if not isinstance(lst, ast.List):
+            raise NotImplementedError(type(lst))
+        values = [make_const(v) for v in items.value]
+        lst.elts.extend(values)
+        return lst
+
+    def LIST_TO_TUPLE(decompiler):
+        tos = decompiler.stack.pop()
+        if not isinstance(tos, ast.List):
+            throw(InvalidQuery, "Translation error, please contact developers: list expected, got: %r" % tos)
+        return ast.Tuple(tos.elts, ast.Load())
 
     def LOAD_ATTR(decompiler, attr_name):
         return ast.Getattr(decompiler.stack.pop(), attr_name)
@@ -427,9 +494,11 @@ def LOAD_FAST(decompiler, varname):
         decompiler.names.add(varname)
         return ast.Name(varname)
 
-    def LOAD_GLOBAL(decompiler, varname):
+    def LOAD_GLOBAL(decompiler, varname, push_null):
+        if push_null:
+            decompiler.stack.append(None)
         decompiler.names.add(varname)
-        return ast.Name(varname)
+        return ast.Name(varname, ast.Load())
 
     def LOAD_METHOD(decompiler, methname):
         return decompiler.LOAD_ATTR(methname)
@@ -470,8 +539,44 @@ def MAKE_FUNCTION(decompiler, argc):
         flags = 0  # todo
         return ast.Lambda(argnames, defaults, flags, func_decompiler.ast)
 
+    def conditional_jump_none_impl(decompiler, endpos, negate):
+        expr = decompiler.stack.pop()
+        assert(decompiler.pos < decompiler.conditions_end)
+        if decompiler.pos in decompiler.or_jumps:
+            clausetype = ast.Or
+            op = ast.IsNot if negate else ast.Is
+        else:
+            clausetype = ast.And
+            op = ast.Is if negate else ast.IsNot
+        expr = ast.Compare(expr, [op()], [ast.Constant(None)])
+        decompiler.stack.append(expr)
+
+        if decompiler.next_pos in decompiler.targets:
+            decompiler.process_target(decompiler.next_pos)
+
+        expr = decompiler.stack.pop()
+        clause = ast.BoolOp(op=clausetype(), values=[expr])
+        clause.endpos = endpos
+        decompiler.targets.setdefault(endpos, clause)
+        return clause
+
+    def jump_if_none(decompiler, endpos):
+        return decompiler.conditional_jump_none_impl(endpos, False)
+
+    def jump_if_not_none(decompiler, endpos):
+        return decompiler.conditional_jump_none_impl(endpos, True)
+
+
+    POP_JUMP_BACKWARD_IF_FALSE = JUMP_IF_FALSE
+    POP_JUMP_BACKWARD_IF_TRUE = JUMP_IF_TRUE
+    POP_JUMP_FORWARD_IF_FALSE = JUMP_IF_FALSE
+    POP_JUMP_FORWARD_IF_TRUE = JUMP_IF_TRUE
     POP_JUMP_IF_FALSE = JUMP_IF_FALSE
     POP_JUMP_IF_TRUE = JUMP_IF_TRUE
+    POP_JUMP_BACKWARD_IF_NONE = jump_if_none
+    POP_JUMP_BACKWARD_IF_NOT_NONE = jump_if_not_none
+    POP_JUMP_FORWARD_IF_NONE = jump_if_none
+    POP_JUMP_FORWARD_IF_NOT_NONE = jump_if_not_none
 
     def POP_TOP(decompiler):
         pass
@@ -483,6 +588,12 @@ def RETURN_VALUE(decompiler):
         decompiler.stack.append(simplify(expr))
         raise AstGenerated()
 
+    def RETURN_GENERATOR(decompiler):
+        pass
+
+    def RESUME(decompiler, where):
+        pass
+
     def ROT_TWO(decompiler):
         tos = decompiler.stack.pop()
         tos1 = decompiler.stack.pop()

requirements.txt

@@ -5,7 +5,7 @@ dateparser==0.7.2
 futures==3.1.1
 decorator==4.2.1
 pymysql==0.9.1
-psycopg2==2.8.4
+psycopg2==2.9.9
 typing-extensions==3.7.4.1
 typeguard==2.9.1
 tzlocal==2.1